git_cache_proxy/
metrics.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Prometheus metrics.
3//!
4//! Cardinality note: both `requests_total` and `upstream_ops_total` carry a
5//! `repo` label so operators can see per-repo traffic and totals
6//! (`sum without (repo) (...)`). To keep the label set bounded, the real repo
7//! name is emitted only for operations that *succeeded* (a served request, a
8//! completed clone/fetch); every failure - failed auth, malformed path,
9//! upstream error, a resolved-but-nonexistent repo - uses a `-` sentinel. The
10//! set of successfully served repos is bounded (the repos the fleet actually
11//! clones), so a flood of distinct but doomed repo paths cannot inflate the
12//! series count.
13
14use prometheus::{
15    Encoder, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge, Opts, Registry,
16    TextEncoder,
17};
18
19/// Histogram buckets, in seconds, for git operation latency: from a fast cached
20/// advertisement (tens of ms) to a large clone over a slow WAN (minutes).
21const DURATION_BUCKETS: &[f64] = &[
22    0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
23];
24
25/// The `kind` label on `requests_total`: which client endpoint served the request.
26#[derive(Debug, Clone, Copy)]
27pub enum RequestKind {
28    InfoRefs,
29    UploadPack,
30    Auth,
31    ReceivePack,
32    LfsBatch,
33    LfsObject,
34}
35
36/// The `result` label on `requests_total` / `upstream_ops_total`.
37#[derive(Debug, Clone, Copy)]
38pub enum Status {
39    Ok,
40    Error,
41    UpstreamError,
42    Unauthorized,
43    Rejected,
44}
45
46/// The `op` label on `upstream_ops_total` / `upstream_duration_seconds`.
47#[derive(Debug, Clone, Copy)]
48pub enum UpstreamOp {
49    Clone,
50    Fetch,
51}
52
53/// The `kind` label on `serve_duration_seconds`.
54#[derive(Debug, Clone, Copy)]
55pub enum ServeKind {
56    InfoRefs,
57    UploadPack,
58}
59
60/// The `result` label on `lfs_objects_total`.
61#[derive(Debug, Clone, Copy)]
62pub enum LfsResult {
63    Hit,
64    Miss,
65    Error,
66}
67
68impl RequestKind {
69    fn as_str(self) -> &'static str {
70        match self {
71            Self::InfoRefs => "info_refs",
72            Self::UploadPack => "upload_pack",
73            Self::Auth => "auth",
74            Self::ReceivePack => "receive_pack",
75            Self::LfsBatch => "lfs_batch",
76            Self::LfsObject => "lfs_object",
77        }
78    }
79}
80
81impl Status {
82    fn as_str(self) -> &'static str {
83        match self {
84            Self::Ok => "ok",
85            Self::Error => "error",
86            Self::UpstreamError => "upstream_error",
87            Self::Unauthorized => "unauthorized",
88            Self::Rejected => "rejected",
89        }
90    }
91}
92
93impl UpstreamOp {
94    fn as_str(self) -> &'static str {
95        match self {
96            Self::Clone => "clone",
97            Self::Fetch => "fetch",
98        }
99    }
100}
101
102impl ServeKind {
103    fn as_str(self) -> &'static str {
104        match self {
105            Self::InfoRefs => "info_refs",
106            Self::UploadPack => "upload_pack",
107        }
108    }
109}
110
111impl LfsResult {
112    fn as_str(self) -> &'static str {
113        match self {
114            Self::Hit => "hit",
115            Self::Miss => "miss",
116            Self::Error => "error",
117        }
118    }
119}
120
121pub struct Metrics {
122    pub registry: Registry,
123    /// `requests_total{kind, result, repo}` - kind = info_refs | upload_pack |
124    /// auth | receive_pack | lfs_batch | lfs_object; result = ok | error |
125    /// upstream_error | unauthorized | rejected; repo = the served repo path when
126    /// result = ok, else `-`.
127    requests: IntCounterVec,
128    /// `upstream_ops_total{op, result, repo}` - op = clone | fetch; result = ok |
129    /// error; repo = the repo path when result = ok, else `-`.
130    upstream: IntCounterVec,
131    /// `cache_bytes` - total size of the on-disk mirror cache, maintained
132    /// incrementally as mirrors are added, refreshed, and evicted. Populated only
133    /// when a cap is configured (`--cache-max-mb`); with no cap it stays `0`.
134    cache_bytes: IntGauge,
135    /// `cache_mirrors` - number of cached mirrors (same caveat as `cache_bytes`).
136    cache_mirrors: IntGauge,
137    /// `evictions_total` - idle mirrors evicted to keep the cache under the cap.
138    evictions: IntCounter,
139    /// `lfs_objects_total{result}` - cached git-LFS object lookups; result = hit
140    /// (served from disk) | miss (fetched from upstream, then cached) | error. No
141    /// `repo` label: objects are content-addressed and shared across repos.
142    lfs_objects: IntCounterVec,
143    /// `upstream_duration_seconds{op, repo}` - clone/fetch wall-clock, observed
144    /// only on success (same bounded-`repo` discipline as the counters).
145    upstream_duration: HistogramVec,
146    /// `serve_duration_seconds{kind, repo}` - kind = info_refs (the buffered
147    /// advertisement) | upload_pack (the packfile stream, timed to EOF). Observed
148    /// only on success.
149    serve_duration: HistogramVec,
150}
151
152impl Metrics {
153    pub fn new() -> Self {
154        let registry = Registry::new();
155        let requests = IntCounterVec::new(
156            Opts::new("gitcacheproxy_requests_total", "Git requests served"),
157            &["kind", "result", "repo"],
158        )
159        .expect("valid metric");
160        let upstream = IntCounterVec::new(
161            Opts::new(
162                "gitcacheproxy_upstream_ops_total",
163                "Upstream clone/fetch operations",
164            ),
165            &["op", "result", "repo"],
166        )
167        .expect("valid metric");
168        let cache_bytes = IntGauge::new(
169            "gitcacheproxy_cache_bytes",
170            "Total size of the on-disk mirror cache in bytes",
171        )
172        .expect("valid metric");
173        let cache_mirrors = IntGauge::new(
174            "gitcacheproxy_cache_mirrors",
175            "Number of cached mirrors on disk",
176        )
177        .expect("valid metric");
178        let evictions = IntCounter::new(
179            "gitcacheproxy_evictions_total",
180            "Idle mirrors evicted to keep the cache under the configured cap",
181        )
182        .expect("valid metric");
183        let lfs_objects = IntCounterVec::new(
184            Opts::new(
185                "gitcacheproxy_lfs_objects_total",
186                "Cached git-LFS object lookups (hit/miss/error)",
187            ),
188            &["result"],
189        )
190        .expect("valid metric");
191        let upstream_duration = HistogramVec::new(
192            HistogramOpts::new(
193                "gitcacheproxy_upstream_duration_seconds",
194                "Upstream clone/fetch duration in seconds",
195            )
196            .buckets(DURATION_BUCKETS.to_vec()),
197            &["op", "repo"],
198        )
199        .expect("valid metric");
200        let serve_duration = HistogramVec::new(
201            HistogramOpts::new(
202                "gitcacheproxy_serve_duration_seconds",
203                "Client serve duration in seconds (info/refs advertisement, upload-pack stream)",
204            )
205            .buckets(DURATION_BUCKETS.to_vec()),
206            &["kind", "repo"],
207        )
208        .expect("valid metric");
209        registry
210            .register(Box::new(requests.clone()))
211            .expect("register requests");
212        registry
213            .register(Box::new(upstream.clone()))
214            .expect("register upstream");
215        registry
216            .register(Box::new(cache_bytes.clone()))
217            .expect("register cache_bytes");
218        registry
219            .register(Box::new(cache_mirrors.clone()))
220            .expect("register cache_mirrors");
221        registry
222            .register(Box::new(evictions.clone()))
223            .expect("register evictions");
224        registry
225            .register(Box::new(lfs_objects.clone()))
226            .expect("register lfs_objects");
227        registry
228            .register(Box::new(upstream_duration.clone()))
229            .expect("register upstream_duration");
230        registry
231            .register(Box::new(serve_duration.clone()))
232            .expect("register serve_duration");
233        Self {
234            registry,
235            requests,
236            upstream,
237            cache_bytes,
238            cache_mirrors,
239            evictions,
240            lfs_objects,
241            upstream_duration,
242            serve_duration,
243        }
244    }
245
246    /// Record the outcome of a client request. `repo` is the served repo path on
247    /// success and `-` on any failure, so unbounded client-supplied paths cannot
248    /// inflate label cardinality.
249    pub fn record_request(&self, kind: RequestKind, result: Status, repo: &str) {
250        self.requests
251            .with_label_values(&[kind.as_str(), result.as_str(), repo])
252            .inc();
253    }
254
255    /// Record an upstream clone/fetch. Errors are recorded too (`Status::Error`);
256    /// pass the repo path on success and `-` on error, matching `record_request`.
257    pub fn record_upstream(&self, op: UpstreamOp, result: Status, repo: &str) {
258        self.upstream
259            .with_label_values(&[op.as_str(), result.as_str(), repo])
260            .inc();
261    }
262
263    /// Refresh the cache-size gauges from the eviction index.
264    pub fn set_cache_size(&self, bytes: u64, mirrors: usize) {
265        self.cache_bytes.set(bytes as i64);
266        self.cache_mirrors.set(mirrors as i64);
267    }
268
269    /// Record one evicted mirror.
270    pub fn record_eviction(&self) {
271        self.evictions.inc();
272    }
273
274    /// Record a cached LFS object lookup (hit / miss / error).
275    pub fn record_lfs(&self, result: LfsResult) {
276        self.lfs_objects.with_label_values(&[result.as_str()]).inc();
277    }
278
279    /// Observe an upstream op's duration. Call only on success with the real repo,
280    /// matching the counters' bounded-`repo` cardinality discipline.
281    pub fn observe_upstream(&self, op: UpstreamOp, repo: &str, seconds: f64) {
282        self.upstream_duration
283            .with_label_values(&[op.as_str(), repo])
284            .observe(seconds);
285    }
286
287    /// Observe a client serve duration, same cardinality discipline as
288    /// `observe_upstream`.
289    pub fn observe_serve(&self, kind: ServeKind, repo: &str, seconds: f64) {
290        self.serve_duration
291            .with_label_values(&[kind.as_str(), repo])
292            .observe(seconds);
293    }
294
295    pub fn gather(&self) -> String {
296        let mut buf = Vec::new();
297        let enc = TextEncoder::new();
298        // encode never fails for the text format into a Vec.
299        let _ = enc.encode(&self.registry.gather(), &mut buf);
300        String::from_utf8_lossy(&buf).into_owned()
301    }
302}
303
304impl Default for Metrics {
305    fn default() -> Self {
306        Self::new()
307    }
308}
309
310#[cfg(test)]
311mod tests {
312    use super::*;
313
314    #[test]
315    fn gather_renders_recorded_series() {
316        let m = Metrics::new();
317        m.record_request(RequestKind::InfoRefs, Status::Ok, "group/foo.git");
318        m.record_request(RequestKind::UploadPack, Status::Error, "group/bar.git");
319        m.record_upstream(UpstreamOp::Fetch, Status::Ok, "group/foo.git");
320        m.record_upstream(UpstreamOp::Clone, Status::Error, "group/bar.git");
321        m.set_cache_size(2048, 3);
322        m.record_eviction();
323        m.record_eviction();
324        m.record_lfs(LfsResult::Hit);
325        m.record_lfs(LfsResult::Miss);
326        m.observe_upstream(UpstreamOp::Clone, "group/foo.git", 1.5);
327        m.observe_serve(ServeKind::UploadPack, "group/foo.git", 2.0);
328
329        let out = m.gather();
330        assert!(out.contains("gitcacheproxy_cache_bytes 2048"));
331        assert!(out.contains("gitcacheproxy_cache_mirrors 3"));
332        assert!(out.contains("gitcacheproxy_evictions_total 2"));
333        assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="hit"} 1"#));
334        assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="miss"} 1"#));
335        assert!(out.contains(
336            r#"gitcacheproxy_upstream_duration_seconds_count{op="clone",repo="group/foo.git"} 1"#
337        ));
338        assert!(out.contains(
339            r#"gitcacheproxy_serve_duration_seconds_count{kind="upload_pack",repo="group/foo.git"} 1"#
340        ));
341        assert!(out.contains(
342            r#"gitcacheproxy_requests_total{kind="info_refs",repo="group/foo.git",result="ok"} 1"#
343        ));
344        assert!(out.contains(
345            r#"gitcacheproxy_requests_total{kind="upload_pack",repo="group/bar.git",result="error"} 1"#
346        ));
347        assert!(out.contains(r#"op="fetch",repo="group/foo.git",result="ok"#));
348        assert!(out.contains(r#"op="clone",repo="group/bar.git",result="error"#));
349    }
350}
git_cache_proxy/metrics.rs

git_cache_proxy/
metrics.rs
