tandem-server 0.6.2

HTTP server for Tandem engine APIs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

use axum::extract::State;
use axum::http::header;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};

use crate::app::state::automation::scheduler::AutomationSchedulerMetrics;
use crate::AppState;

pub(super) async fn prometheus_metrics(State(state): State<AppState>) -> Response {
    if !crate::config::env::prometheus_metrics_enabled() {
        return StatusCode::NOT_FOUND.into_response();
    }

    let scheduler = state.automation_scheduler.read().await.metrics();
    let mut body = render_scheduler_metrics(&scheduler);
    body.push_str(&tandem_observability::render_observability_metrics_prometheus());

    (
        [(
            header::CONTENT_TYPE,
            "text/plain; version=0.0.4; charset=utf-8",
        )],
        body,
    )
        .into_response()
}

fn render_scheduler_metrics(metrics: &AutomationSchedulerMetrics) -> String {
    let mut out = String::new();
    render_metric_line(
        &mut out,
        "tandem_scheduler_active_runs",
        &[],
        metrics.active_runs as u64,
    );
    render_metric_line(
        &mut out,
        "tandem_scheduler_admitted_total",
        &[],
        metrics.admitted_total,
    );
    render_metric_line(
        &mut out,
        "tandem_scheduler_completed_total",
        &[],
        metrics.completed_total,
    );
    render_metric_line(
        &mut out,
        "tandem_scheduler_queue_wait_ms_avg",
        &[],
        metrics.avg_wait_ms,
    );
    render_metric_line(
        &mut out,
        "tandem_scheduler_queue_wait_ms_p95",
        &[],
        metrics.p95_wait_ms,
    );
    for (reason, count) in &metrics.queued_runs_by_reason {
        render_metric_line(
            &mut out,
            "tandem_scheduler_queued_runs",
            &[("reason", reason.as_str())],
            *count as u64,
        );
    }
    out
}

fn render_metric_line(out: &mut String, name: &str, labels: &[(&str, &str)], value: u64) {
    out.push_str(name);
    if !labels.is_empty() {
        out.push('{');
        for (idx, (key, value)) in labels.iter().enumerate() {
            if idx > 0 {
                out.push(',');
            }
            out.push_str(key);
            out.push_str("=\"");
            out.push_str(&escape_label_value(value));
            out.push('"');
        }
        out.push('}');
    }
    out.push(' ');
    out.push_str(&value.to_string());
    out.push('\n');
}

fn escape_label_value(value: &str) -> String {
    value
        .replace('\\', "\\\\")
        .replace('\n', "\\n")
        .replace('"', "\\\"")
}