codoseo_web/
metrics.rs

1//! The metrics CodoSEO records, named and labelled in one place. The `metrics` facade does
2//! nothing until a recorder is installed (the `codoseo` binary installs a Prometheus one for
3//! its server roles when `CODOSEO_METRICS_BIND` is set), so library code and tests can call
4//! these freely.
5//!
6//! Labels are a closed set of short words (a lane number, an outcome, a channel kind). Never
7//! put a URL, a domain, an email or an id in one: every distinct value is a new time series.
8
9use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
10
11pub const CRAWL_QUEUE_WAIT_SECONDS: &str = "codoseo_crawl_queue_wait_seconds";
12pub const CRAWLS_RUNNING: &str = "codoseo_crawls_running";
13pub const CRAWLS_FINISHED_TOTAL: &str = "codoseo_crawls_finished_total";
14pub const PAGES_CRAWLED_TOTAL: &str = "codoseo_pages_crawled_total";
15pub const WORKER_MEMORY_BUDGET_BYTES: &str = "codoseo_worker_memory_budget_bytes";
16pub const WORKER_MEMORY_RESERVED_BYTES: &str = "codoseo_worker_memory_reserved_bytes";
17pub const DB_POOL_CONNECTIONS: &str = "codoseo_db_pool_connections";
18pub const ALERT_DELIVERIES_TOTAL: &str = "codoseo_alert_deliveries_total";
19pub const API_REQUESTS_TOTAL: &str = "codoseo_api_requests_total";
20pub const SCHEDULER_LAST_TICK_TIMESTAMP_SECONDS: &str =
21    "codoseo_scheduler_last_tick_timestamp_seconds";
22pub const JOBS_FAILED_TOTAL: &str = "codoseo_jobs_failed_total";
23
24/// Every metric name, for the tests and the docs.
25pub const ALL: [&str; 11] = [
26    CRAWL_QUEUE_WAIT_SECONDS,
27    CRAWLS_RUNNING,
28    CRAWLS_FINISHED_TOTAL,
29    PAGES_CRAWLED_TOTAL,
30    WORKER_MEMORY_BUDGET_BYTES,
31    WORKER_MEMORY_RESERVED_BYTES,
32    DB_POOL_CONNECTIONS,
33    ALERT_DELIVERIES_TOTAL,
34    API_REQUESTS_TOTAL,
35    SCHEDULER_LAST_TICK_TIMESTAMP_SECONDS,
36    JOBS_FAILED_TOTAL,
37];
38
39/// Crawl priorities run 0 (most urgent) to 5; the label is the lane.
40pub const LANES: [&str; 6] = ["0", "1", "2", "3", "4", "5"];
41/// The alert channel kinds (`ChannelKind::as_str`).
42pub const CHANNELS: [&str; 4] = ["email", "slack", "discord", "webhook"];
43/// The job kinds (`JobKind::slug`).
44pub const JOB_KINDS: [&str; 4] = ["send_alert", "send_digest", "send_email", "cleanup"];
45/// What `api_requests_total`'s `result` is: `ok`, an [`AgentError::code`](crate::agent::error::AgentError::code) or `error`.
46pub const API_RESULTS: [&str; 10] = [
47    "ok",
48    "unauthorized",
49    "not_found",
50    "bad_request",
51    "crawl_in_progress",
52    "plan_limit",
53    "quota_exceeded",
54    "unavailable",
55    "internal",
56    // A no-key MCP tool call that failed (those errors carry no code).
57    "error",
58];
59
60/// Which API an agent call came through.
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub enum Surface {
63    Rest,
64    Mcp,
65}
66
67impl Surface {
68    fn label(self) -> &'static str {
69        match self {
70            Surface::Rest => "rest",
71            Surface::Mcp => "mcp",
72        }
73    }
74}
75
76/// Whether the caller sent an API key (`key`) or is one of the no-key MCP clients (`anon`).
77#[derive(Debug, Clone, Copy, PartialEq, Eq)]
78pub enum Tier {
79    Key,
80    Anon,
81}
82
83impl Tier {
84    fn label(self) -> &'static str {
85        match self {
86            Tier::Key => "key",
87            Tier::Anon => "anon",
88        }
89    }
90}
91
92/// Describes every metric and creates the series that are known up front at zero, so a scrape
93/// right after startup already lists them (and `rate()` sees the first increment).
94pub fn register() {
95    describe_histogram!(
96        CRAWL_QUEUE_WAIT_SECONDS,
97        metrics::Unit::Seconds,
98        "Seconds a crawl waited in the queue before a worker claimed it, by priority lane"
99    );
100    describe_gauge!(CRAWLS_RUNNING, "Crawls this process is running right now");
101    describe_counter!(
102        CRAWLS_FINISHED_TOTAL,
103        "Crawl runs that ended, by outcome (completed or failed; a failed first attempt is retried)"
104    );
105    describe_counter!(PAGES_CRAWLED_TOTAL, "Pages fetched by crawls");
106    describe_gauge!(
107        WORKER_MEMORY_BUDGET_BYTES,
108        metrics::Unit::Bytes,
109        "Memory the worker may spend on crawls (70% of its cgroup limit)"
110    );
111    describe_gauge!(
112        WORKER_MEMORY_RESERVED_BYTES,
113        metrics::Unit::Bytes,
114        "Memory reserved by the crawls running now (page cap times the per-page estimate)"
115    );
116    describe_gauge!(
117        DB_POOL_CONNECTIONS,
118        "Postgres pool connections, by state (idle or active)"
119    );
120    describe_counter!(
121        ALERT_DELIVERIES_TOTAL,
122        "Alert delivery attempts, by channel kind and result"
123    );
124    describe_counter!(
125        API_REQUESTS_TOTAL,
126        "Agent API calls, by surface (rest, mcp), tier (key, anon) and result (ok or an error code)"
127    );
128    describe_gauge!(
129        SCHEDULER_LAST_TICK_TIMESTAMP_SECONDS,
130        metrics::Unit::Seconds,
131        "Unix time of the scheduler's last tick"
132    );
133    describe_counter!(JOBS_FAILED_TOTAL, "Job runs that failed, by job kind");
134
135    gauge!(CRAWLS_RUNNING).set(0.0);
136    gauge!(WORKER_MEMORY_RESERVED_BYTES).set(0.0);
137    counter!(PAGES_CRAWLED_TOTAL).absolute(0);
138    for outcome in ["completed", "failed"] {
139        counter!(CRAWLS_FINISHED_TOTAL, "outcome" => outcome).absolute(0);
140    }
141    for state in ["idle", "active"] {
142        gauge!(DB_POOL_CONNECTIONS, "state" => state).set(0.0);
143    }
144    for channel in CHANNELS {
145        for result in ["ok", "error"] {
146            counter!(ALERT_DELIVERIES_TOTAL, "channel" => channel, "result" => result).absolute(0);
147        }
148    }
149    for kind in JOB_KINDS {
150        counter!(JOBS_FAILED_TOTAL, "kind" => kind).absolute(0);
151    }
152    for surface in [Surface::Rest, Surface::Mcp] {
153        for tier in [Tier::Key, Tier::Anon] {
154            counter!(API_REQUESTS_TOTAL, "surface" => surface.label(), "tier" => tier.label(), "result" => "ok")
155                .absolute(0);
156        }
157    }
158    for lane in LANES {
159        let _ = histogram!(CRAWL_QUEUE_WAIT_SECONDS, "lane" => lane);
160    }
161}
162
163/// A claimed crawl waited `seconds` in the queue. `priority` is the crawl's priority (its lane).
164pub fn queue_wait(priority: i16, seconds: f64) {
165    let lane = LANES[usize::try_from(priority).unwrap_or(0).min(LANES.len() - 1)];
166    histogram!(CRAWL_QUEUE_WAIT_SECONDS, "lane" => lane).record(seconds.max(0.0));
167}
168
169/// Holds one running crawl in the gauges: `crawls_running` goes up by one and the reserved
170/// memory by `reserved_bytes`, both undone when this is dropped (a panic included).
171#[must_use = "the crawl counts as running until this is dropped"]
172pub struct RunningCrawl {
173    reserved_bytes: f64,
174}
175
176/// Marks a crawl as running, reserving `reserved_bytes` of the worker's memory budget.
177pub fn crawl_started(reserved_bytes: u64) -> RunningCrawl {
178    let reserved_bytes = reserved_bytes as f64;
179    gauge!(CRAWLS_RUNNING).increment(1.0);
180    gauge!(WORKER_MEMORY_RESERVED_BYTES).increment(reserved_bytes);
181    RunningCrawl { reserved_bytes }
182}
183
184impl Drop for RunningCrawl {
185    fn drop(&mut self) {
186        gauge!(CRAWLS_RUNNING).decrement(1.0);
187        gauge!(WORKER_MEMORY_RESERVED_BYTES).decrement(self.reserved_bytes);
188    }
189}
190
191/// A crawl run ended: `completed` when its results were stored, otherwise `failed`.
192pub fn crawl_finished(completed: bool) {
193    let outcome = if completed { "completed" } else { "failed" };
194    counter!(CRAWLS_FINISHED_TOTAL, "outcome" => outcome).increment(1);
195}
196
197pub fn pages_crawled(pages: u64) {
198    counter!(PAGES_CRAWLED_TOTAL).increment(pages);
199}
200
201pub fn worker_memory_budget(bytes: u64) {
202    gauge!(WORKER_MEMORY_BUDGET_BYTES).set(bytes as f64);
203}
204
205/// The pool's connections right now: `idle` ones are free, `active` ones are checked out.
206pub fn db_pool(idle: u32, active: u32) {
207    gauge!(DB_POOL_CONNECTIONS, "state" => "idle").set(f64::from(idle));
208    gauge!(DB_POOL_CONNECTIONS, "state" => "active").set(f64::from(active));
209}
210
211/// One delivery attempt on a channel of kind `channel` (`ChannelKind::as_str`).
212pub fn alert_delivery(channel: &'static str, ok: bool) {
213    let result = if ok { "ok" } else { "error" };
214    counter!(ALERT_DELIVERIES_TOTAL, "channel" => channel, "result" => result).increment(1);
215}
216
217/// One agent API call that ended with `result` (`ok` or an error code).
218pub fn api_request(surface: Surface, tier: Tier, result: &'static str) {
219    counter!(API_REQUESTS_TOTAL, "surface" => surface.label(), "tier" => tier.label(), "result" => result)
220        .increment(1);
221}
222
223pub fn scheduler_tick(unix_seconds: f64) {
224    gauge!(SCHEDULER_LAST_TICK_TIMESTAMP_SECONDS).set(unix_seconds);
225}
226
227/// A job of kind `kind` (`JobKind::slug`) failed or panicked.
228pub fn job_failed(kind: &'static str) {
229    counter!(JOBS_FAILED_TOTAL, "kind" => kind).increment(1);
230}
codoseo_web/metrics.rs

codoseo_web/
metrics.rs
