1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
//! Prometheus metrics endpoint.
//!
//! Exposes all collected metrics in the Prometheus text exposition format.
//! Only available when the `prometheus` feature is enabled.
//!
//! # Endpoint
//!
//! | Method | Path | Description |
//! |--------|------|-------------|
//! | `GET` | `/api/v1/metrics` | Returns metrics in Prometheus text format |
//!
//! # Metrics exposed
//!
//! All metrics defined in [`ironflow_core::metric_names`] are available,
//! including run counters, step histograms, agent cost/token gauges,
//! worker activity, and API request latency.
//!
//! # Grafana dashboard
//!
//! Recommended layout: three rows (Overview, Steps & Operations, Worker),
//! each with 2-4 panels. All queries assume the default Prometheus data
//! source and a `$job` variable set to the scrape job name.
//!
//! ## Row 1 -- Overview
//!
//! ```promql
//! # Panel: Run throughput (completed runs per minute, by workflow)
//! sum by (workflow) (rate(ironflow_runs_total{status="Completed"}[5m]))
//!
//! # Panel: Run failure rate (percentage)
//! sum(rate(ironflow_runs_total{status="Failed"}[5m]))
//! / sum(rate(ironflow_runs_total[5m])) * 100
//!
//! # Panel: P50 / P95 / P99 run duration
//! histogram_quantile(0.50, rate(ironflow_run_duration_seconds_bucket[5m]))
//! histogram_quantile(0.95, rate(ironflow_run_duration_seconds_bucket[5m]))
//! histogram_quantile(0.99, rate(ironflow_run_duration_seconds_bucket[5m]))
//!
//! # Panel: Active runs (gauge)
//! ironflow_runs_active
//! ```
//!
//! ## Row 2 -- Steps & Operations
//!
//! ```promql
//! # Panel: Step throughput by kind
//! sum by (kind) (rate(ironflow_steps_total[5m]))
//!
//! # Panel: Agent cost rate (USD per hour, by model)
//! sum by (model) (rate(ironflow_agent_cost_usd_total[1h]))
//!
//! # Panel: Agent token throughput (input + output per minute)
//! rate(ironflow_agent_tokens_input_total[5m])
//! rate(ironflow_agent_tokens_output_total[5m])
//!
//! # Panel: Shell / HTTP step duration P95
//! histogram_quantile(0.95, rate(ironflow_shell_duration_seconds_bucket[5m]))
//! histogram_quantile(0.95, rate(ironflow_http_duration_seconds_bucket[5m]))
//! ```
//!
//! ## Row 3 -- Worker
//!
//! ```promql
//! # Panel: Queue depth (pending runs waiting for a worker)
//! ironflow_worker_queue_depth
//!
//! # Panel: Active worker tasks
//! ironflow_worker_active
//!
//! # Panel: Poll hit/miss rate
//! rate(ironflow_worker_polls_total{result="hit"}[5m])
//! rate(ironflow_worker_polls_total{result="miss"}[5m])
//!
//! # Panel: Lease losses (counter, alerts if > 0)
//! increase(ironflow_worker_leases_lost_total[1h])
//! ```
//!
//! ## Row 4 -- API
//!
//! ```promql
//! # Panel: API request rate by path
//! sum by (path) (rate(ironflow_api_requests_total[5m]))
//!
//! # Panel: API latency P95 by path
//! histogram_quantile(0.95, sum by (le, path)
//! (rate(ironflow_api_request_duration_seconds_bucket[5m])))
//! ```
//!
//! ## Alerting rules
//!
//! ```promql
//! # Alert: high run failure rate (> 10% over 15 minutes)
//! sum(rate(ironflow_runs_total{status="Failed"}[15m]))
//! / sum(rate(ironflow_runs_total[15m])) > 0.10
//!
//! # Alert: queue depth growing (> 10 pending runs for 5 minutes)
//! ironflow_worker_queue_depth > 10
//!
//! # Alert: no worker activity (zero polls for 5 minutes)
//! rate(ironflow_worker_polls_total[5m]) == 0
//!
//! # Alert: budget exceeded (any run cancelled for cost)
//! increase(ironflow_run_budget_exceeded_total[1h]) > 0
//! ```
use State;
use CONTENT_TYPE;
use ;
use PrometheusHandle;
/// Handler for `GET /api/v1/metrics`.
///
/// Returns all registered Prometheus metrics in text exposition format.
///
/// # Examples
///
/// ```no_run
/// use axum::Router;
/// use axum::routing::get;
/// use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
///
/// # fn example() {
/// let handle = PrometheusBuilder::new()
/// .install_recorder()
/// .expect("failed to install recorder");
///
/// let app: Router<PrometheusHandle> = Router::new()
/// .route("/metrics", get(ironflow_api::routes::metrics::metrics));
/// # }
/// ```
pub async