#[cfg(feature = "telemetry")]
use std::fmt;
#[cfg(feature = "telemetry")]
use crate::reload::ConfigStatus;
#[cfg(feature = "tracing")]
use crate::reload::ReloadReason;
#[cfg(feature = "telemetry")]
use crate::remote::RemoteStatus;
#[cfg(feature = "tracing")]
pub(crate) fn installed<T: ?Sized>(
reason: &ReloadReason,
generation: u64,
) -> tracing::span::EnteredSpan {
let span = tracing::info_span!(
target: "dynamic_config",
"dynamic_config.reload",
config = std::any::type_name::<T>(),
reason = reason.as_str(),
generation = generation,
outcome = "installed",
);
let entered = span.entered();
tracing::info!(
target: "dynamic_config",
"a configuration snapshot was installed"
);
entered
}
#[cfg(feature = "tracing")]
pub(crate) fn refused<T: ?Sized>(error: &crate::Error) {
tracing::warn!(
target: "dynamic_config",
config = std::any::type_name::<T>(),
outcome = "rejected",
error.kind = error.kind().as_str(),
error.path = error.path().as_str(),
"a reload installed nothing; the previous snapshot is still serving"
);
}
#[cfg(feature = "tracing")]
pub(crate) fn fetching() -> tracing::span::EnteredSpan {
fetching_span().entered()
}
#[cfg(all(feature = "tracing", feature = "async"))]
pub(crate) fn fetching_async() -> tracing::Span {
fetching_span()
}
#[cfg(feature = "tracing")]
fn fetching_span() -> tracing::Span {
tracing::info_span!(
target: "dynamic_config",
"dynamic_config.fetch",
outcome = tracing::field::Empty,
)
}
#[cfg(feature = "tracing")]
pub(crate) fn fetched(span: &tracing::Span, elapsed: std::time::Duration) {
span.record("outcome", "fetched");
span.in_scope(|| {
tracing::info!(
target: "dynamic_config",
outcome = "fetched",
duration_ms = elapsed.as_secs_f64() * 1000.0,
"a remote store answered"
);
});
}
#[cfg(feature = "tracing")]
pub(crate) fn fetch_failed(span: &tracing::Span, error: &crate::Error) {
span.record("outcome", "failed");
span.in_scope(|| {
tracing::warn!(
target: "dynamic_config",
outcome = "failed",
error.kind = error.kind().as_str(),
"a fetch from a remote store returned nothing; the document it \
last answered with is still in the slot"
);
});
}
#[cfg(feature = "telemetry")]
pub const INSTALLS_TOTAL: &str = "dynamic_config_installs_total";
#[cfg(feature = "telemetry")]
pub const LAST_SUCCESS_SECONDS: &str = "dynamic_config_last_success_seconds";
#[cfg(feature = "telemetry")]
pub const CONSECUTIVE_FAILURES: &str = "dynamic_config_consecutive_failures";
#[cfg(feature = "telemetry")]
pub const LAST_FAILURE_SECONDS: &str = "dynamic_config_last_failure_seconds";
#[cfg(feature = "telemetry")]
pub const LAST_RELOAD_INFO: &str = "dynamic_config_last_reload_info";
#[cfg(feature = "telemetry")]
pub const LAST_FAILURE_INFO: &str = "dynamic_config_last_failure_info";
#[cfg(feature = "telemetry")]
pub const METRIC_NAMES: [&str; 6] = [
INSTALLS_TOTAL,
LAST_SUCCESS_SECONDS,
CONSECUTIVE_FAILURES,
LAST_FAILURE_SECONDS,
LAST_RELOAD_INFO,
LAST_FAILURE_INFO,
];
#[cfg(feature = "telemetry")]
pub const REMOTE_UP: &str = "dynamic_config_remote_up";
#[cfg(feature = "telemetry")]
pub const REMOTE_FETCHES_TOTAL: &str = "dynamic_config_remote_fetches_total";
#[cfg(feature = "telemetry")]
pub const REMOTE_LAST_FETCH_SECONDS: &str = "dynamic_config_remote_last_fetch_seconds";
#[cfg(feature = "telemetry")]
pub const REMOTE_LAST_FETCH_DURATION_SECONDS: &str =
"dynamic_config_remote_last_fetch_duration_seconds";
#[cfg(feature = "telemetry")]
pub const REMOTE_CONSECUTIVE_FAILURES: &str = "dynamic_config_remote_consecutive_failures";
#[cfg(feature = "telemetry")]
pub const REMOTE_LAST_FAILURE_INFO: &str = "dynamic_config_remote_last_failure_info";
#[cfg(feature = "telemetry")]
pub const REMOTE_METRIC_NAMES: [&str; 6] = [
REMOTE_UP,
REMOTE_FETCHES_TOTAL,
REMOTE_LAST_FETCH_SECONDS,
REMOTE_LAST_FETCH_DURATION_SECONDS,
REMOTE_CONSECUTIVE_FAILURES,
REMOTE_LAST_FAILURE_INFO,
];
#[cfg(feature = "telemetry")]
#[cfg_attr(docsrs, doc(cfg(feature = "telemetry")))]
#[derive(Debug, Clone, Default)]
pub struct Exposition {
entries: Vec<Entry>,
remotes: Vec<RemoteEntry>,
}
#[cfg(feature = "telemetry")]
#[derive(Debug, Clone)]
struct Entry {
labels: String,
status: ConfigStatus,
}
#[cfg(feature = "telemetry")]
#[derive(Debug, Clone)]
struct RemoteEntry {
labels: String,
status: RemoteStatus,
}
#[cfg(feature = "telemetry")]
impl Exposition {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn add(&mut self, config: &str, status: &ConfigStatus) {
self.add_with(&[("config", config)], status);
}
pub fn add_with(&mut self, labels: &[(&str, &str)], status: &ConfigStatus) {
self.entries.push(Entry {
labels: render_labels(labels),
status: status.clone(),
});
}
pub fn add_remote(&mut self, config: &str, status: &RemoteStatus) {
self.add_remote_with(&[("config", config)], status);
}
pub fn add_remote_with(&mut self, labels: &[(&str, &str)], status: &RemoteStatus) {
self.remotes.push(RemoteEntry {
labels: render_labels(labels),
status: status.clone(),
});
}
#[must_use]
pub fn render(&self) -> String {
let mut out = String::new();
self.family(
&mut out,
INSTALLS_TOTAL,
"counter",
"Configuration snapshots installed since the process started.",
|entry| Some((String::new(), entry.status.generation.to_string())),
);
self.family(
&mut out,
LAST_SUCCESS_SECONDS,
"gauge",
"Seconds since the serving configuration snapshot was installed.",
|entry| {
entry
.status
.stale_for()
.map(|elapsed| (String::new(), seconds(elapsed)))
},
);
self.family(
&mut out,
CONSECUTIVE_FAILURES,
"gauge",
"Reloads that installed nothing since one did; zero is healthy.",
|entry| Some((String::new(), entry.status.consecutive_failures.to_string())),
);
self.family(
&mut out,
LAST_FAILURE_SECONDS,
"gauge",
"Seconds since the last reload that installed nothing.",
|entry| {
entry
.status
.last_failure
.as_ref()
.map(|failure| (String::new(), seconds(failure.at.elapsed())))
},
);
self.family(
&mut out,
LAST_RELOAD_INFO,
"gauge",
"Why the serving snapshot was installed; always 1.",
|entry| {
entry
.status
.last_reason
.as_ref()
.map(|reason| (format!("reason=\"{}\"", reason.as_str()), "1".to_owned()))
},
);
self.family(
&mut out,
LAST_FAILURE_INFO,
"gauge",
"The category of the last reload that installed nothing; always 1.",
|entry| {
entry.status.last_failure.as_ref().map(|failure| {
(
format!("kind=\"{}\"", failure.kind.as_str()),
"1".to_owned(),
)
})
},
);
self.remote_families(&mut out);
out
}
fn remote_families(&self, out: &mut String) {
remote_family(
out,
&self.remotes,
REMOTE_UP,
"gauge",
"Whether the remote store answered the last time it was asked; \
absent until it has been.",
|status| {
status
.reachable()
.map(|up| (String::new(), u8::from(up).to_string()))
},
);
remote_family(
out,
&self.remotes,
REMOTE_FETCHES_TOTAL,
"counter",
"Documents a remote store has handed over since the process started.",
|status| Some((String::new(), status.fetches.to_string())),
);
remote_family(
out,
&self.remotes,
REMOTE_LAST_FETCH_SECONDS,
"gauge",
"Seconds since a remote store last handed a document over.",
|status| {
status
.stale_for()
.map(|elapsed| (String::new(), seconds(elapsed)))
},
);
remote_family(
out,
&self.remotes,
REMOTE_LAST_FETCH_DURATION_SECONDS,
"gauge",
"How long the last pulled fetch took, in seconds.",
|status| {
status
.last_fetch_duration
.map(|elapsed| (String::new(), seconds(elapsed)))
},
);
remote_family(
out,
&self.remotes,
REMOTE_CONSECUTIVE_FAILURES,
"gauge",
"Fetches that returned nothing since one returned a document; zero is healthy.",
|status| Some((String::new(), status.consecutive_failures.to_string())),
);
remote_family(
out,
&self.remotes,
REMOTE_LAST_FAILURE_INFO,
"gauge",
"The category of the last fetch that returned nothing; always 1.",
|status| {
status.last_failure.as_ref().map(|failure| {
(
format!("kind=\"{}\"", failure.kind.as_str()),
"1".to_owned(),
)
})
},
);
}
fn family(
&self,
out: &mut String,
name: &str,
kind: &str,
help: &str,
sample: impl Fn(&Entry) -> Option<(String, String)>,
) {
let samples: Vec<_> = self
.entries
.iter()
.filter_map(|entry| {
sample(entry).map(|(extra, value)| (join(&entry.labels, &extra), value))
})
.collect();
write_family(out, name, kind, help, &samples);
}
}
#[cfg(feature = "telemetry")]
fn remote_family(
out: &mut String,
entries: &[RemoteEntry],
name: &str,
kind: &str,
help: &str,
sample: impl Fn(&RemoteStatus) -> Option<(String, String)>,
) {
let samples: Vec<_> = entries
.iter()
.filter_map(|entry| {
sample(&entry.status).map(|(extra, value)| (join(&entry.labels, &extra), value))
})
.collect();
write_family(out, name, kind, help, &samples);
}
#[cfg(feature = "telemetry")]
fn write_family(
out: &mut String,
name: &str,
kind: &str,
help: &str,
samples: &[(String, String)],
) {
if samples.is_empty() {
return;
}
out.push_str("# HELP ");
out.push_str(name);
out.push(' ');
out.push_str(help);
out.push_str("\n# TYPE ");
out.push_str(name);
out.push(' ');
out.push_str(kind);
out.push('\n');
for (labels, value) in samples {
out.push_str(name);
if !labels.is_empty() {
out.push('{');
out.push_str(labels);
out.push('}');
}
out.push(' ');
out.push_str(value);
out.push('\n');
}
}
#[cfg(feature = "telemetry")]
fn render_labels(labels: &[(&str, &str)]) -> String {
let mut rendered = String::new();
for (name, value) in labels {
if !rendered.is_empty() {
rendered.push(',');
}
push_label_name(&mut rendered, name);
rendered.push_str("=\"");
push_label_value(&mut rendered, value);
rendered.push('"');
}
rendered
}
#[cfg(feature = "telemetry")]
impl fmt::Display for Exposition {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.render())
}
}
#[cfg(feature = "telemetry")]
fn join(left: &str, right: &str) -> String {
match (left.is_empty(), right.is_empty()) {
(true, _) => right.to_owned(),
(_, true) => left.to_owned(),
_ => format!("{left},{right}"),
}
}
#[cfg(feature = "telemetry")]
fn seconds(elapsed: std::time::Duration) -> String {
format!("{:.3}", elapsed.as_secs_f64())
}
#[cfg(feature = "telemetry")]
fn push_label_name(out: &mut String, name: &str) {
let start = out.len();
for character in name.chars() {
if character.is_ascii_alphanumeric() || character == '_' {
out.push(character);
} else {
out.push('_');
}
}
match out[start..].chars().next() {
None => out.push('_'),
Some(first) if first.is_ascii_digit() => out.insert(start, '_'),
Some(_) => {}
}
}
#[cfg(feature = "telemetry")]
fn push_label_value(out: &mut String, value: &str) {
for character in value.chars() {
match character {
'\\' => out.push_str(r"\\"),
'"' => out.push_str("\\\""),
'\n' => out.push_str("\\n"),
_ => out.push(character),
}
}
}
#[cfg(all(test, feature = "telemetry"))]
mod tests {
use super::*;
#[test]
fn a_label_name_is_coerced_and_a_value_is_escaped() {
let mut name = String::new();
push_label_name(&mut name, "pod-name");
assert_eq!(name, "pod_name");
let mut leading = String::new();
push_label_name(&mut leading, "9lives");
assert_eq!(leading, "_9lives");
let mut empty = String::new();
push_label_name(&mut empty, "");
assert_eq!(empty, "_");
let mut value = String::new();
push_label_value(&mut value, "x\" } 1\nforged 2");
assert_eq!(value, r#"x\" } 1\nforged 2"#);
}
#[test]
fn labels_join_from_either_side_or_neither() {
assert_eq!(join("a=\"1\"", "b=\"2\""), "a=\"1\",b=\"2\"");
assert_eq!(join("", "b=\"2\""), "b=\"2\"");
assert_eq!(join("a=\"1\"", ""), "a=\"1\"");
assert_eq!(join("", ""), "");
}
}