use prometheus::{
Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Opts,
Registry,
};
use std::sync::{LazyLock, OnceLock};
static METRICS_INIT: OnceLock<Result<(), MetricsError>> = OnceLock::new();
#[derive(Debug, Clone)]
pub struct MetricsError {
pub message: String,
}
impl std::fmt::Display for MetricsError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Metrics initialization failed: {}", self.message)
}
}
impl std::error::Error for MetricsError {}
fn latency_histogram_opts(name: &str, help: &str) -> HistogramOpts {
HistogramOpts::new(name, help).buckets(vec![
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
])
}
fn fast_histogram_opts(name: &str, help: &str) -> HistogramOpts {
HistogramOpts::new(name, help).buckets(vec![
0.0001, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05,
])
}
pub static METRICS_REGISTRY: LazyLock<Registry> = LazyLock::new(Registry::new);
pub static HTTP_REQUEST_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
latency_histogram_opts(
"shodh_http_request_duration_seconds",
"HTTP request duration in seconds",
),
&["method", "endpoint", "status"],
)
.expect("HTTP_REQUEST_DURATION metric must be valid at compile time")
});
pub static HTTP_REQUESTS_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new("shodh_http_requests_total", "Total HTTP requests"),
&["method", "endpoint", "status"],
)
.expect("HTTP_REQUESTS_TOTAL metric must be valid at compile time")
});
pub static MEMORY_STORE_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new("shodh_memory_store_total", "Total memory store operations"),
&["result"],
)
.expect("MEMORY_STORE_TOTAL metric must be valid at compile time")
});
pub static MEMORY_STORE_DURATION: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_memory_store_duration_seconds",
"Memory store operation duration",
)
.buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5]),
)
.expect("MEMORY_STORE_DURATION metric must be valid at compile time")
});
pub static MEMORY_RETRIEVE_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_memory_retrieve_total",
"Total memory retrieve operations",
),
&["retrieval_mode", "result"],
)
.expect("MEMORY_RETRIEVE_TOTAL metric must be valid at compile time")
});
pub static MEMORY_RETRIEVE_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"shodh_memory_retrieve_duration_seconds",
"Memory retrieve operation duration",
)
.buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]),
&["retrieval_mode"],
)
.expect("MEMORY_RETRIEVE_DURATION metric must be valid at compile time")
});
pub static MEMORY_RETRIEVE_RESULTS: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"shodh_memory_retrieve_results",
"Number of results returned per query",
)
.buckets(vec![0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0]),
&["retrieval_mode"],
)
.expect("MEMORY_RETRIEVE_RESULTS metric must be valid at compile time")
});
pub static ONTOLOGICAL_INTENT_CONFIDENCE: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_ontological_intent_confidence",
"Distribution of inferred ontological intent confidence scores",
)
.buckets(vec![0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),
)
.expect("ONTOLOGICAL_INTENT_CONFIDENCE metric must be valid at compile time")
});
pub static ONTOLOGICAL_RERANK_BOOST_APPLIED: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_ontological_rerank_boost",
"Distribution of ontological re-rank boost values applied to memories",
)
.buckets(vec![0.0, 0.02, 0.04, 0.08, 0.12, 0.16, 0.20, 0.25]),
)
.expect("ONTOLOGICAL_RERANK_BOOST_APPLIED metric must be valid at compile time")
});
pub static ONTOLOGICAL_FALLBACK_TOTAL: LazyLock<IntCounter> = LazyLock::new(|| {
IntCounter::new(
"shodh_ontological_fallback_total",
"Queries where ontological intent was below confidence threshold",
)
.expect("ONTOLOGICAL_FALLBACK_TOTAL metric must be valid at compile time")
});
pub static ONTOLOGICAL_DENSITY_SKIP_TOTAL: LazyLock<IntCounter> = LazyLock::new(|| {
IntCounter::new(
"shodh_ontological_density_skip_total",
"Queries where ontological filtering was disabled due to high graph density",
)
.expect("ONTOLOGICAL_DENSITY_SKIP_TOTAL metric must be valid at compile time")
});
pub static EMBEDDING_GENERATE_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_embedding_generate_total",
"Total embedding generations",
),
&["mode", "result"], )
.expect("EMBEDDING_GENERATE_TOTAL metric must be valid at compile time")
});
pub static EMBEDDING_GENERATE_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"shodh_embedding_generate_duration_seconds",
"Embedding generation duration",
)
.buckets(vec![
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0,
]),
&["mode"],
)
.expect("EMBEDDING_GENERATE_DURATION metric must be valid at compile time")
});
pub static EMBEDDING_TIMEOUT_TOTAL: LazyLock<IntCounter> = LazyLock::new(|| {
IntCounter::new(
"shodh_embedding_timeout_total",
"Total embedding generation timeouts",
)
.expect("EMBEDDING_TIMEOUT_TOTAL metric must be valid at compile time")
});
pub static NER_LOCK_TIMEOUT_TOTAL: LazyLock<IntCounter> = LazyLock::new(|| {
IntCounter::new(
"shodh_ner_lock_timeout_total",
"Total NER session lock timeouts (degraded entity extraction)",
)
.expect("NER_LOCK_TIMEOUT_TOTAL metric must be valid at compile time")
});
pub static ACTIVE_USERS: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_active_users",
"Number of users with active memory sessions",
)
.expect("ACTIVE_USERS metric must be valid at compile time")
});
pub static MEMORIES_BY_TIER: LazyLock<IntGaugeVec> = LazyLock::new(|| {
IntGaugeVec::new(
Opts::new("shodh_memories_by_tier", "Total memories by tier"),
&["tier"], )
.expect("MEMORIES_BY_TIER metric must be valid at compile time")
});
pub static MEMORY_HEAP_BYTES_TOTAL: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_memory_heap_bytes_total",
"Total estimated heap usage across all users",
)
.expect("MEMORY_HEAP_BYTES_TOTAL metric must be valid at compile time")
});
pub static VECTOR_INDEX_SIZE_TOTAL: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_vector_index_size_total",
"Total number of vectors in all indices",
)
.expect("VECTOR_INDEX_SIZE_TOTAL metric must be valid at compile time")
});
pub static VECTOR_SEARCH_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_vector_search_total",
"Total vector search operations",
),
&["result"],
)
.expect("VECTOR_SEARCH_TOTAL metric must be valid at compile time")
});
pub static VECTOR_SEARCH_DURATION: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(fast_histogram_opts(
"shodh_vector_search_duration_seconds",
"Vector search duration",
))
.expect("VECTOR_SEARCH_DURATION metric must be valid at compile time")
});
pub static ROCKSDB_OPS_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new("shodh_rocksdb_ops_total", "Total RocksDB operations"),
&["operation", "result"], )
.expect("ROCKSDB_OPS_TOTAL metric must be valid at compile time")
});
pub static ROCKSDB_OPS_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
fast_histogram_opts(
"shodh_rocksdb_ops_duration_seconds",
"RocksDB operation duration",
),
&["operation"],
)
.expect("ROCKSDB_OPS_DURATION metric must be valid at compile time")
});
pub static LEGACY_FALLBACK_BRANCH_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_legacy_fallback_branch_total",
"Total fallback deserialization branch hits",
),
&["branch"],
)
.expect("LEGACY_FALLBACK_BRANCH_TOTAL metric must be valid at compile time")
});
pub static ERRORS_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new("shodh_errors_total", "Total errors by type"),
&["error_type", "endpoint"],
)
.expect("ERRORS_TOTAL metric must be valid at compile time")
});
pub static RESOURCE_LIMIT_REJECTIONS: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_resource_limit_rejections",
"Requests rejected due to resource limits",
),
&["resource"],
)
.expect("RESOURCE_LIMIT_REJECTIONS metric must be valid at compile time")
});
pub static CONCURRENT_REQUESTS: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_concurrent_requests",
"Current number of concurrent requests",
)
.expect("CONCURRENT_REQUESTS metric must be valid at compile time")
});
pub static REQUEST_QUEUE_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new("shodh_request_queue_size", "Number of queued requests")
.expect("REQUEST_QUEUE_SIZE metric must be valid at compile time")
});
pub static HEBBIAN_REINFORCE_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_hebbian_reinforce_total",
"Total Hebbian reinforcement operations",
),
&["outcome", "result"], )
.expect("HEBBIAN_REINFORCE_TOTAL metric must be valid at compile time")
});
pub static HEBBIAN_REINFORCE_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"shodh_hebbian_reinforce_duration_seconds",
"Hebbian reinforcement operation duration",
)
.buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5]),
&["outcome"],
)
.expect("HEBBIAN_REINFORCE_DURATION metric must be valid at compile time")
});
pub static CONSOLIDATE_TOTAL: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_consolidate_total",
"Total memory consolidation operations",
),
&["result"],
)
.expect("CONSOLIDATE_TOTAL metric must be valid at compile time")
});
pub static CONSOLIDATE_DURATION: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_consolidate_duration_seconds",
"Memory consolidation operation duration",
)
.buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]),
)
.expect("CONSOLIDATE_DURATION metric must be valid at compile time")
});
pub static BATCH_STORE_DURATION: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_batch_store_duration_seconds",
"Batch memory store operation duration",
)
.buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]),
)
.expect("BATCH_STORE_DURATION metric must be valid at compile time")
});
pub static BATCH_STORE_SIZE: LazyLock<Histogram> = LazyLock::new(|| {
Histogram::with_opts(
HistogramOpts::new(
"shodh_batch_store_size",
"Number of memories in batch store operations",
)
.buckets(vec![
1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0,
]),
)
.expect("BATCH_STORE_SIZE metric must be valid at compile time")
});
pub static EMBEDDING_CACHE_QUERY: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_embedding_cache_query_total",
"Query embedding cache operations",
),
&["result"], )
.expect("EMBEDDING_CACHE_QUERY metric must be valid at compile time")
});
pub static EMBEDDING_CACHE_CONTENT: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"shodh_embedding_cache_content_total",
"Content embedding cache operations",
),
&["result"], )
.expect("EMBEDDING_CACHE_CONTENT metric must be valid at compile time")
});
pub static EMBEDDING_CACHE_QUERY_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_embedding_cache_query_size",
"Current number of entries in query embedding cache",
)
.expect("EMBEDDING_CACHE_QUERY_SIZE metric must be valid at compile time")
});
pub static EMBEDDING_CACHE_CONTENT_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"shodh_embedding_cache_content_size",
"Current number of entries in content embedding cache",
)
.expect("EMBEDDING_CACHE_CONTENT_SIZE metric must be valid at compile time")
});
pub fn register_metrics() -> Result<(), MetricsError> {
if let Some(result) = METRICS_INIT.get() {
return result.clone();
}
let result = do_register_metrics();
let _ = METRICS_INIT.set(result.clone());
result
}
fn do_register_metrics() -> Result<(), MetricsError> {
let mut errors = Vec::new();
macro_rules! register {
($metric:expr, $name:expr) => {
if let Err(e) = METRICS_REGISTRY.register(Box::new($metric.clone())) {
errors.push(format!("{}: {}", $name, e));
}
};
}
register!(HTTP_REQUEST_DURATION, "HTTP_REQUEST_DURATION");
register!(HTTP_REQUESTS_TOTAL, "HTTP_REQUESTS_TOTAL");
register!(MEMORY_STORE_TOTAL, "MEMORY_STORE_TOTAL");
register!(MEMORY_STORE_DURATION, "MEMORY_STORE_DURATION");
register!(MEMORY_RETRIEVE_TOTAL, "MEMORY_RETRIEVE_TOTAL");
register!(MEMORY_RETRIEVE_DURATION, "MEMORY_RETRIEVE_DURATION");
register!(MEMORY_RETRIEVE_RESULTS, "MEMORY_RETRIEVE_RESULTS");
register!(
ONTOLOGICAL_INTENT_CONFIDENCE,
"ONTOLOGICAL_INTENT_CONFIDENCE"
);
register!(
ONTOLOGICAL_RERANK_BOOST_APPLIED,
"ONTOLOGICAL_RERANK_BOOST_APPLIED"
);
register!(ONTOLOGICAL_FALLBACK_TOTAL, "ONTOLOGICAL_FALLBACK_TOTAL");
register!(
ONTOLOGICAL_DENSITY_SKIP_TOTAL,
"ONTOLOGICAL_DENSITY_SKIP_TOTAL"
);
register!(EMBEDDING_GENERATE_TOTAL, "EMBEDDING_GENERATE_TOTAL");
register!(EMBEDDING_GENERATE_DURATION, "EMBEDDING_GENERATE_DURATION");
register!(EMBEDDING_TIMEOUT_TOTAL, "EMBEDDING_TIMEOUT_TOTAL");
register!(NER_LOCK_TIMEOUT_TOTAL, "NER_LOCK_TIMEOUT_TOTAL");
register!(ACTIVE_USERS, "ACTIVE_USERS");
register!(MEMORIES_BY_TIER, "MEMORIES_BY_TIER");
register!(MEMORY_HEAP_BYTES_TOTAL, "MEMORY_HEAP_BYTES_TOTAL");
register!(VECTOR_INDEX_SIZE_TOTAL, "VECTOR_INDEX_SIZE_TOTAL");
register!(VECTOR_SEARCH_TOTAL, "VECTOR_SEARCH_TOTAL");
register!(VECTOR_SEARCH_DURATION, "VECTOR_SEARCH_DURATION");
register!(ROCKSDB_OPS_TOTAL, "ROCKSDB_OPS_TOTAL");
register!(ROCKSDB_OPS_DURATION, "ROCKSDB_OPS_DURATION");
register!(LEGACY_FALLBACK_BRANCH_TOTAL, "LEGACY_FALLBACK_BRANCH_TOTAL");
register!(ERRORS_TOTAL, "ERRORS_TOTAL");
register!(RESOURCE_LIMIT_REJECTIONS, "RESOURCE_LIMIT_REJECTIONS");
register!(CONCURRENT_REQUESTS, "CONCURRENT_REQUESTS");
register!(REQUEST_QUEUE_SIZE, "REQUEST_QUEUE_SIZE");
register!(HEBBIAN_REINFORCE_TOTAL, "HEBBIAN_REINFORCE_TOTAL");
register!(HEBBIAN_REINFORCE_DURATION, "HEBBIAN_REINFORCE_DURATION");
register!(CONSOLIDATE_TOTAL, "CONSOLIDATE_TOTAL");
register!(CONSOLIDATE_DURATION, "CONSOLIDATE_DURATION");
register!(BATCH_STORE_DURATION, "BATCH_STORE_DURATION");
register!(BATCH_STORE_SIZE, "BATCH_STORE_SIZE");
register!(EMBEDDING_CACHE_QUERY, "EMBEDDING_CACHE_QUERY");
register!(EMBEDDING_CACHE_CONTENT, "EMBEDDING_CACHE_CONTENT");
register!(EMBEDDING_CACHE_QUERY_SIZE, "EMBEDDING_CACHE_QUERY_SIZE");
register!(EMBEDDING_CACHE_CONTENT_SIZE, "EMBEDDING_CACHE_CONTENT_SIZE");
if errors.is_empty() {
Ok(())
} else {
Err(MetricsError {
message: errors.join("; "),
})
}
}
pub struct Timer {
histogram: Histogram,
start: std::time::Instant,
}
impl Timer {
pub fn new(histogram: Histogram) -> Self {
Self {
histogram,
start: std::time::Instant::now(),
}
}
}
impl Drop for Timer {
fn drop(&mut self) {
let duration = self.start.elapsed().as_secs_f64();
self.histogram.observe(duration);
}
}
#[cfg(test)]
mod tests {
use super::*;
use prometheus::core::Metric;
#[test]
fn test_metrics_registration_is_idempotent() {
let result1 = register_metrics();
let result2 = register_metrics();
assert_eq!(result1.is_ok(), result2.is_ok());
}
#[test]
fn test_timer_records_duration() {
let histogram = Histogram::with_opts(HistogramOpts::new(
"test_timer_histogram",
"Test histogram for timer",
))
.unwrap();
{
let _timer = Timer::new(histogram.clone());
std::thread::sleep(std::time::Duration::from_millis(10));
}
let metric = histogram.metric();
assert_eq!(metric.get_histogram().get_sample_count(), 1);
assert!(metric.get_histogram().get_sample_sum() >= 0.01);
}
}