#![cfg_attr(
not(test),
deny(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::todo,
clippy::unimplemented,
clippy::indexing_slicing,
clippy::string_slice,
clippy::arithmetic_side_effects,
)
)]
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use serde::Serialize;
use crate::shadow::diff::{Comparison, Divergence};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
pub struct ShadowStats {
pub mirrored: u64,
pub compared: u64,
pub matched: u64,
pub diverged: u64,
pub shadow_errors: u64,
pub primary_errors: u64,
pub shadow_timeouts: u64,
pub dropped_at_capacity: u64,
pub skipped_oversize: u64,
pub skipped_refused: u64,
pub skipped_conditional: u64,
pub primary_incomplete: u64,
pub comparisons_abandoned: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RequestContext {
pub method: String,
pub target: String,
pub route: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct DivergenceRecord {
pub method: String,
pub target: String,
pub route: String,
pub occurrences: u64,
pub first_observed_at_ms: u64,
pub last_observed_at_ms: u64,
#[serde(flatten)]
pub divergence: Divergence,
}
const MAX_ROUTE_SERIES: usize = 200;
pub const OVERFLOW_ROUTE_LABEL: &str = "__other__";
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct LabelledCount {
pub route: String,
pub label: String,
pub count: u64,
}
#[derive(Debug, Default)]
struct LabelledSeries {
counts: std::collections::BTreeMap<(String, String), u64>,
routes: std::collections::BTreeSet<String>,
}
impl LabelledSeries {
fn record(&mut self, route: &str, label: &str) {
let route = if self.routes.contains(route) {
route.to_owned()
} else if self.routes.len() < MAX_ROUTE_SERIES {
self.routes.insert(route.to_owned());
route.to_owned()
} else {
OVERFLOW_ROUTE_LABEL.to_owned()
};
let entry = self.counts.entry((route, label.to_owned())).or_insert(0);
*entry = entry.saturating_add(1);
}
fn snapshot(&self) -> Vec<LabelledCount> {
self.counts
.iter()
.map(|((route, label), count)| LabelledCount {
route: route.clone(),
label: label.clone(),
count: *count,
})
.collect()
}
}
#[derive(Clone, Debug, Serialize)]
pub struct ShadowSnapshot {
pub enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub target: Option<String>,
pub stats: ShadowStats,
pub comparisons_by_route: Vec<LabelledCount>,
pub divergences_by_route: Vec<LabelledCount>,
pub divergences: Vec<DivergenceRecord>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Recorded {
Match,
NewDivergence(Box<Divergence>),
RepeatDivergence,
}
impl ShadowSnapshot {
#[must_use]
pub fn disabled() -> Self {
Self {
enabled: false,
target: None,
stats: ShadowStats::default(),
comparisons_by_route: Vec::new(),
divergences_by_route: Vec::new(),
divergences: Vec::new(),
}
}
}
#[derive(Clone, Debug)]
pub struct ShadowRegistry {
counters: Arc<Counters>,
records: Arc<RwLock<VecDeque<DivergenceRecord>>>,
comparisons_by_route: Arc<RwLock<LabelledSeries>>,
divergences_by_route: Arc<RwLock<LabelledSeries>>,
max_records: usize,
}
#[derive(Debug, Default)]
struct Counters {
mirrored: AtomicU64,
compared: AtomicU64,
matched: AtomicU64,
diverged: AtomicU64,
shadow_errors: AtomicU64,
primary_errors: AtomicU64,
shadow_timeouts: AtomicU64,
dropped_at_capacity: AtomicU64,
skipped_oversize: AtomicU64,
skipped_refused: AtomicU64,
skipped_conditional: AtomicU64,
primary_incomplete: AtomicU64,
comparisons_abandoned: AtomicU64,
}
#[allow(
deprecated,
reason = "`fetch_update` is renamed `try_update` on rustc 1.99, but `try_update` is newer than the 1.88 MSRV"
)]
fn bump(counter: &AtomicU64) {
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_add(1))
});
}
impl ShadowRegistry {
#[must_use]
pub fn new(max_records: usize) -> Self {
Self {
counters: Arc::new(Counters::default()),
records: Arc::new(RwLock::new(VecDeque::new())),
comparisons_by_route: Arc::new(RwLock::new(LabelledSeries::default())),
divergences_by_route: Arc::new(RwLock::new(LabelledSeries::default())),
max_records: max_records.max(1),
}
}
pub fn record_outcome(&self, route: &str, outcome: &str) {
if let Ok(mut series) = self.comparisons_by_route.write() {
series.record(route, outcome);
}
}
pub fn record_divergence_kind(&self, route: &str, kind: &str) {
if let Ok(mut series) = self.divergences_by_route.write() {
series.record(route, kind);
}
}
#[must_use]
pub fn comparisons_by_route(&self) -> Vec<LabelledCount> {
self.comparisons_by_route
.read()
.map(|series| series.snapshot())
.unwrap_or_default()
}
#[must_use]
pub fn divergences_by_route(&self) -> Vec<LabelledCount> {
self.divergences_by_route
.read()
.map(|series| series.snapshot())
.unwrap_or_default()
}
pub fn record_mirrored(&self) {
bump(&self.counters.mirrored);
}
pub fn record_dropped_at_capacity(&self) {
bump(&self.counters.dropped_at_capacity);
}
pub fn record_shadow_error(&self) {
bump(&self.counters.shadow_errors);
}
pub fn record_primary_error(&self) {
bump(&self.counters.primary_errors);
}
pub fn record_shadow_timeout(&self) {
bump(&self.counters.shadow_timeouts);
}
pub fn record_skipped_oversize(&self) {
bump(&self.counters.skipped_oversize);
}
pub fn record_skipped_refused(&self) {
bump(&self.counters.skipped_refused);
}
pub fn record_skipped_conditional(&self) {
bump(&self.counters.skipped_conditional);
}
pub fn record_primary_incomplete(&self) {
bump(&self.counters.primary_incomplete);
}
pub fn record_comparison_abandoned(&self) {
bump(&self.counters.comparisons_abandoned);
}
#[must_use = "the caller decides whether to log; ignore it with `let _ =` when not"]
pub fn record_comparison(
&self,
context: &RequestContext,
comparison: Comparison,
observed_at_ms: u64,
) -> Recorded {
bump(&self.counters.compared);
let divergence = match comparison {
Comparison::Match => {
bump(&self.counters.matched);
return Recorded::Match;
}
Comparison::Diverged(divergence) => *divergence,
};
bump(&self.counters.diverged);
let Ok(mut records) = self.records.write() else {
return Recorded::RepeatDivergence;
};
if let Some(existing) = records.iter_mut().find(|record| {
record.divergence.fingerprint == divergence.fingerprint && record.route == context.route
}) {
existing.occurrences = existing.occurrences.saturating_add(1);
existing.last_observed_at_ms = observed_at_ms;
return Recorded::RepeatDivergence;
}
while records.len() >= self.max_records {
records.pop_front();
}
records.push_back(DivergenceRecord {
method: context.method.clone(),
target: context.target.clone(),
route: context.route.clone(),
occurrences: 1,
first_observed_at_ms: observed_at_ms,
last_observed_at_ms: observed_at_ms,
divergence: divergence.clone(),
});
Recorded::NewDivergence(Box::new(divergence))
}
#[must_use]
pub fn stats(&self) -> ShadowStats {
ShadowStats {
mirrored: self.counters.mirrored.load(Ordering::Relaxed),
compared: self.counters.compared.load(Ordering::Relaxed),
matched: self.counters.matched.load(Ordering::Relaxed),
diverged: self.counters.diverged.load(Ordering::Relaxed),
shadow_errors: self.counters.shadow_errors.load(Ordering::Relaxed),
primary_errors: self.counters.primary_errors.load(Ordering::Relaxed),
shadow_timeouts: self.counters.shadow_timeouts.load(Ordering::Relaxed),
dropped_at_capacity: self.counters.dropped_at_capacity.load(Ordering::Relaxed),
skipped_oversize: self.counters.skipped_oversize.load(Ordering::Relaxed),
skipped_refused: self.counters.skipped_refused.load(Ordering::Relaxed),
skipped_conditional: self.counters.skipped_conditional.load(Ordering::Relaxed),
primary_incomplete: self.counters.primary_incomplete.load(Ordering::Relaxed),
comparisons_abandoned: self.counters.comparisons_abandoned.load(Ordering::Relaxed),
}
}
#[must_use]
pub fn recent(&self) -> Vec<DivergenceRecord> {
self.records
.read()
.map(|records| records.iter().cloned().collect())
.unwrap_or_default()
}
#[must_use]
pub fn snapshot(&self, enabled: bool, target: Option<&str>) -> ShadowSnapshot {
ShadowSnapshot {
enabled,
target: target.map(ToOwned::to_owned),
stats: self.stats(),
comparisons_by_route: self.comparisons_by_route(),
divergences_by_route: self.divergences_by_route(),
divergences: self.recent(),
}
}
#[cfg(test)]
fn pin_mirrored_at_ceiling_for_tests(&self) {
self.counters.mirrored.store(u64::MAX, Ordering::Relaxed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::log::filter::ParameterFilter;
use crate::shadow::diff::{Comparison, ResponseFacts, compare};
use bytes::Bytes;
fn facts(status: u16, body: &str) -> ResponseFacts {
ResponseFacts::new(
status,
Some("application/json".to_owned()),
Bytes::from(body.to_owned()),
)
}
fn context() -> RequestContext {
RequestContext {
method: "GET".to_owned(),
target: "/api/orders".to_owned(),
route: "/api/*".to_owned(),
}
}
fn diverging(primary: &str, shadow: &str) -> Comparison {
compare(
&facts(200, primary),
&facts(200, shadow),
&ParameterFilter::default(),
2048,
)
}
#[test]
fn a_fresh_registry_is_empty() {
let registry = ShadowRegistry::new(10);
let stats = registry.stats();
assert_eq!(stats.mirrored, 0);
assert_eq!(stats.compared, 0);
assert_eq!(stats.diverged, 0);
assert!(registry.recent().is_empty());
}
#[test]
fn a_match_is_counted_but_not_recorded() {
let registry = ShadowRegistry::new(10);
let _ = registry.record_comparison(&context(), diverging("{\"a\":1}", "{\"a\":1}"), 1_000);
let stats = registry.stats();
assert_eq!(stats.compared, 1);
assert_eq!(stats.matched, 1);
assert_eq!(stats.diverged, 0);
assert!(registry.recent().is_empty());
}
#[test]
fn a_divergence_is_counted_and_recorded() {
let registry = ShadowRegistry::new(10);
let _ = registry.record_comparison(&context(), diverging("{\"a\":1}", "{}"), 1_000);
let stats = registry.stats();
assert_eq!(stats.compared, 1);
assert_eq!(stats.matched, 0);
assert_eq!(stats.diverged, 1);
let recent = registry.recent();
assert_eq!(recent.len(), 1);
assert_eq!(recent[0].method, "GET");
assert_eq!(recent[0].route, "/api/*");
assert_eq!(recent[0].occurrences, 1);
assert_eq!(recent[0].first_observed_at_ms, 1_000);
assert_eq!(recent[0].last_observed_at_ms, 1_000);
}
#[test]
fn repeat_divergences_collapse_by_fingerprint() {
let registry = ShadowRegistry::new(10);
for at in [1_000, 2_000, 3_000] {
let _ = registry.record_comparison(&context(), diverging("{\"a\":1}", "{}"), at);
}
let recent = registry.recent();
assert_eq!(
recent.len(),
1,
"the same divergence must not fill the ring"
);
assert_eq!(recent[0].occurrences, 3);
assert_eq!(recent[0].first_observed_at_ms, 1_000);
assert_eq!(recent[0].last_observed_at_ms, 3_000);
assert_eq!(registry.stats().diverged, 3);
}
#[test]
fn the_record_ring_is_bounded() {
let registry = ShadowRegistry::new(3);
for n in 0_u64..10 {
let _ = registry.record_comparison(
&context(),
diverging(&format!("{{\"a\":{n}}}"), "{}"),
1_000 + n,
);
}
let recent = registry.recent();
assert_eq!(recent.len(), 3, "ring must be capped at max_records");
assert_eq!(registry.stats().diverged, 10, "counters are not capped");
assert_eq!(recent[2].last_observed_at_ms, 1_009);
}
#[test]
fn identical_divergences_on_different_routes_stay_separate() {
let registry = ShadowRegistry::new(10);
for route in ["/api/orders", "/api/users"] {
let context = RequestContext {
method: "GET".to_owned(),
target: route.to_owned(),
route: route.to_owned(),
};
let _ = registry.record_comparison(&context, diverging("{\"a\":1}", "{}"), 1_000);
}
let recent = registry.recent();
assert_eq!(recent.len(), 2);
assert_eq!(recent[0].route, "/api/orders");
assert_eq!(recent[1].route, "/api/users");
assert_eq!(
recent[0].divergence.fingerprint,
recent[1].divergence.fingerprint
);
}
#[test]
fn recording_reports_whether_a_divergence_was_new() {
let registry = ShadowRegistry::new(10);
assert_eq!(
registry.record_comparison(&context(), diverging("{\"a\":1}", "{\"a\":1}"), 1_000),
Recorded::Match
);
let first = registry.record_comparison(&context(), diverging("{\"a\":1}", "{}"), 1_000);
assert!(
matches!(first, Recorded::NewDivergence(_)),
"the first sighting must be reported as new so the layer logs once"
);
assert_eq!(
registry.record_comparison(&context(), diverging("{\"a\":1}", "{}"), 2_000),
Recorded::RepeatDivergence
);
}
#[test]
fn operational_counters_move_independently() {
let registry = ShadowRegistry::new(10);
registry.record_mirrored();
registry.record_mirrored();
registry.record_dropped_at_capacity();
registry.record_shadow_error();
registry.record_shadow_timeout();
registry.record_skipped_oversize();
registry.record_skipped_refused();
registry.record_skipped_conditional();
registry.record_primary_incomplete();
registry.record_comparison_abandoned();
let stats = registry.stats();
assert_eq!(stats.comparisons_abandoned, 1);
assert_eq!(stats.skipped_refused, 1);
assert_eq!(stats.skipped_conditional, 1);
assert_eq!(stats.primary_incomplete, 1);
assert_eq!(stats.mirrored, 2);
assert_eq!(stats.dropped_at_capacity, 1);
assert_eq!(stats.shadow_errors, 1);
assert_eq!(stats.shadow_timeouts, 1);
assert_eq!(stats.skipped_oversize, 1);
assert_eq!(stats.compared, 0);
}
#[test]
fn clones_share_one_registry() {
let registry = ShadowRegistry::new(10);
let clone = registry.clone();
clone.record_mirrored();
assert_eq!(registry.stats().mirrored, 1);
}
#[test]
fn the_snapshot_serializes_the_shape_the_actuator_publishes() {
let registry = ShadowRegistry::new(10);
registry.record_mirrored();
let _ = registry.record_comparison(&context(), diverging("{\"a\":1}", "{}"), 1_000);
let snapshot = registry.snapshot(true, Some("http://127.0.0.1:9091"));
let json = serde_json::to_value(&snapshot).expect("must serialize");
assert_eq!(json["enabled"], serde_json::json!(true));
assert_eq!(json["target"], serde_json::json!("http://127.0.0.1:9091"));
assert_eq!(json["stats"]["diverged"], serde_json::json!(1));
assert_eq!(json["divergences"][0]["route"], serde_json::json!("/api/*"));
assert_eq!(
json["divergences"][0]["kind"],
serde_json::json!("body"),
"the divergence detail must be flattened into the record"
);
}
#[test]
fn a_disabled_snapshot_reports_no_target() {
let registry = ShadowRegistry::new(10);
let json = serde_json::to_value(registry.snapshot(false, None)).expect("must serialize");
assert_eq!(json["enabled"], serde_json::json!(false));
assert!(json.get("target").is_none() || json["target"].is_null());
}
#[test]
fn counters_saturate_rather_than_wrapping() {
let registry = ShadowRegistry::new(10);
registry.pin_mirrored_at_ceiling_for_tests();
registry.record_mirrored();
assert_eq!(registry.stats().mirrored, u64::MAX);
}
}