sz_orm_observability/
lib.rs

1//! SZ-ORM 可观测性模块
2//!
3//! 提供 Prometheus exporter、SLO 燃烧率监控等能力。
4//! OTLP 导出由 `sz-orm-tracing` 的 `otlp` feature 提供（本模块不含 OTLP）。
5//!
6//! # 核心能力
7//!
8//! ## 1. MetricsRegistry（默认启用）
9//!
10//! 统一的指标注册中心，支持 Counter / Gauge / Histogram 三种类型，
11//! 内置线程安全（`RwLock`），可通过 `render()` 输出 Prometheus 文本格式。
12//!
13//! ## 2. Prometheus exporter
14//!
15//! 通过 `start_metrics_server` 在指定端口暴露 `/metrics` HTTP 端点，
16//! 供 Prometheus 拉取。基于 `tokio::net::TcpListener` 实现（不依赖 hyper）。
17//!
18//! ## 3. SLO 燃烧率
19//!
20//! 基于 5m / 1h 两个窗口计算 SLO 燃烧率，支持多窗口告警。
21//!
22//! # 快速入门
23//!
24//! ```no_run
25//! use sz_orm_observability::{MetricsRegistry, MetricKind};
26//! use std::time::Duration;
27//!
28//! // 创建指标注册中心
29//! let registry = MetricsRegistry::new();
30//!
31//! // 注册指标
32//! let counter = registry.register_counter("sz_orm_pool_acquires_total", "Total pool acquire calls");
33//! let gauge = registry.register_gauge("sz_orm_pool_active_connections", "Current active connections");
34//! let histogram = registry.register_histogram(
35//!     "sz_orm_query_duration_seconds",
36//!     "Query duration in seconds",
37//!     vec![0.001, 0.01, 0.1, 1.0, 10.0],
38//! );
39//!
40//! // 更新指标
41//! counter.inc();
42//! gauge.set(5.0);
43//! histogram.observe(0.025);
44//!
45//! // 输出 Prometheus 文本格式
46//! let output = registry.render();
47//! println!("{}", output);
48//! ```
49
50#![warn(missing_docs)]
51
52use parking_lot::RwLock;
53use std::collections::HashMap;
54use std::sync::Arc;
55
56pub mod slo;
57pub mod summary;
58
59#[cfg(feature = "service-mesh")]
60pub mod service_mesh;
61
62#[cfg(feature = "query-logging")]
63pub mod query_logger;
64
65#[cfg(feature = "anomaly-detection")]
66pub mod anomaly;
67
68#[cfg(feature = "anomaly-remediation-rca")]
69#[allow(missing_docs)]
70pub mod anomaly_remediation_rca;
71
72pub use slo::{SloBurnRate, SloConfig, SloMonitor};
73pub use summary::{
74    LabeledHistogram, PushSnapshot, PushgatewayConfig, PushgatewayExporter, Summary,
75};
76
77#[cfg(feature = "query-logging")]
78pub use query_logger::{mask_params, LogLevel, QueryLogEntry, QueryLogger};
79
80/// 指标类型
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub enum MetricKind {
83    /// 单调递增计数器（如总请求数）
84    Counter,
85    /// 可增可减的瞬时值（如当前连接数）
86    Gauge,
87    /// 直方图（如请求延迟分布）
88    Histogram,
89}
90
91/// 指标元数据
92#[derive(Debug, Clone)]
93pub struct MetricMeta {
94    /// 指标名（如 `sz_orm_pool_acquires_total`）
95    pub name: String,
96    /// 帮助文本
97    pub help: String,
98    /// 指标类型
99    pub kind: MetricKind,
100}
101
102/// 计数器（单调递增）
103pub struct Counter {
104    name: String,
105    value: Arc<RwLock<f64>>,
106    labels: HashMap<String, String>,
107}
108
109impl Counter {
110    /// 递增 1
111    pub fn inc(&self) {
112        self.inc_by(1.0);
113    }
114
115    /// 递增指定值
116    pub fn inc_by(&self, delta: f64) {
117        let mut v = self.value.write();
118        *v += delta;
119    }
120
121    /// 当前值
122    pub fn value(&self) -> f64 {
123        *self.value.read()
124    }
125
126    /// 指标名
127    pub fn name(&self) -> &str {
128        &self.name
129    }
130
131    /// 渲染为 Prometheus 文本格式
132    pub fn render(&self) -> String {
133        let v = self.value.read();
134        if self.labels.is_empty() {
135            format!("{} {}\n", self.name, v)
136        } else {
137            let labels: Vec<String> = self
138                .labels
139                .iter()
140                .map(|(k, val)| format!("{}=\"{}\"", k, val.replace('"', "\\\"")))
141                .collect();
142            format!("{}{{{}}} {}\n", self.name, labels.join(","), v)
143        }
144    }
145}
146
147/// Gauge（可增可减）
148pub struct Gauge {
149    name: String,
150    value: Arc<RwLock<f64>>,
151    labels: HashMap<String, String>,
152}
153
154impl Gauge {
155    /// 设置值
156    pub fn set(&self, value: f64) {
157        *self.value.write() = value;
158    }
159
160    /// 递增
161    pub fn inc(&self) {
162        self.inc_by(1.0);
163    }
164
165    /// 递增指定值
166    pub fn inc_by(&self, delta: f64) {
167        let mut v = self.value.write();
168        *v += delta;
169    }
170
171    /// 递减指定值
172    pub fn dec_by(&self, delta: f64) {
173        let mut v = self.value.write();
174        *v -= delta;
175    }
176
177    /// 当前值
178    pub fn value(&self) -> f64 {
179        *self.value.read()
180    }
181
182    /// 指标名
183    pub fn name(&self) -> &str {
184        &self.name
185    }
186
187    /// 渲染为 Prometheus 文本格式
188    pub fn render(&self) -> String {
189        let v = self.value.read();
190        if self.labels.is_empty() {
191            format!("{} {}\n", self.name, v)
192        } else {
193            let labels: Vec<String> = self
194                .labels
195                .iter()
196                .map(|(k, val)| format!("{}=\"{}\"", k, val.replace('"', "\\\"")))
197                .collect();
198            format!("{}{{{}}} {}\n", self.name, labels.join(","), v)
199        }
200    }
201}
202
203/// 直方图（延迟分布等）
204pub struct Histogram {
205    name: String,
206    buckets: Vec<f64>,
207    counts: Arc<RwLock<Vec<u64>>>,
208    sum: Arc<RwLock<f64>>,
209    count: Arc<RwLock<u64>>,
210}
211
212impl Histogram {
213    /// 观察一个值
214    pub fn observe(&self, value: f64) {
215        let mut counts = self.counts.write();
216        for (i, bucket) in self.buckets.iter().enumerate() {
217            if value <= *bucket {
218                counts[i] += 1;
219            }
220        }
221        // 最后一个 bucket 是 +Inf，必须递增
222        let last = counts.len() - 1;
223        counts[last] += 1;
224
225        let mut sum = self.sum.write();
226        *sum += value;
227        let mut count = self.count.write();
228        *count += 1;
229    }
230
231    /// 总观察次数
232    pub fn count(&self) -> u64 {
233        *self.count.read()
234    }
235
236    /// 所有观察值之和
237    pub fn sum(&self) -> f64 {
238        *self.sum.read()
239    }
240
241    /// 指标名
242    pub fn name(&self) -> &str {
243        &self.name
244    }
245
246    /// 渲染为 Prometheus 文本格式
247    pub fn render(&self) -> String {
248        let counts = self.counts.read();
249        let sum = self.sum.read();
250        let count = self.count.read();
251
252        let mut output = String::new();
253        for (i, bucket) in self.buckets.iter().enumerate() {
254            output.push_str(&format!(
255                "{}_bucket{{le=\"{}\"}} {}\n",
256                self.name, bucket, counts[i]
257            ));
258        }
259        output.push_str(&format!("{}_sum {}\n", self.name, sum));
260        output.push_str(&format!("{}_count {}\n", self.name, count));
261        output
262    }
263}
264
265/// 指标注册中心
266pub struct MetricsRegistry {
267    counters: RwLock<HashMap<String, Arc<Counter>>>,
268    gauges: RwLock<HashMap<String, Arc<Gauge>>>,
269    histograms: RwLock<HashMap<String, Arc<Histogram>>>,
270    metas: RwLock<Vec<MetricMeta>>,
271}
272
273impl Default for MetricsRegistry {
274    fn default() -> Self {
275        Self::new()
276    }
277}
278
279impl MetricsRegistry {
280    /// 创建空注册中心
281    pub fn new() -> Self {
282        Self {
283            counters: RwLock::new(HashMap::new()),
284            gauges: RwLock::new(HashMap::new()),
285            histograms: RwLock::new(HashMap::new()),
286            metas: RwLock::new(Vec::new()),
287        }
288    }
289
290    /// 注册 Counter
291    pub fn register_counter(&self, name: &str, help: &str) -> Arc<Counter> {
292        self.register_counter_with_labels(name, help, HashMap::new())
293    }
294
295    /// 注册带标签的 Counter
296    pub fn register_counter_with_labels(
297        &self,
298        name: &str,
299        help: &str,
300        labels: HashMap<String, String>,
301    ) -> Arc<Counter> {
302        let mut counters = self.counters.write();
303        let key = format!("{}_{:?}", name, labels);
304        if let Some(c) = counters.get(&key) {
305            return c.clone();
306        }
307        let counter = Arc::new(Counter {
308            name: name.to_string(),
309            value: Arc::new(RwLock::new(0.0)),
310            labels,
311        });
312        counters.insert(key, counter.clone());
313
314        let mut metas = self.metas.write();
315        metas.push(MetricMeta {
316            name: name.to_string(),
317            help: help.to_string(),
318            kind: MetricKind::Counter,
319        });
320        counter
321    }
322
323    /// 注册 Gauge
324    pub fn register_gauge(&self, name: &str, help: &str) -> Arc<Gauge> {
325        self.register_gauge_with_labels(name, help, HashMap::new())
326    }
327
328    /// 注册带标签的 Gauge
329    pub fn register_gauge_with_labels(
330        &self,
331        name: &str,
332        help: &str,
333        labels: HashMap<String, String>,
334    ) -> Arc<Gauge> {
335        let mut gauges = self.gauges.write();
336        let key = format!("{}_{:?}", name, labels);
337        if let Some(g) = gauges.get(&key) {
338            return g.clone();
339        }
340        let gauge = Arc::new(Gauge {
341            name: name.to_string(),
342            value: Arc::new(RwLock::new(0.0)),
343            labels,
344        });
345        gauges.insert(key, gauge.clone());
346
347        let mut metas = self.metas.write();
348        metas.push(MetricMeta {
349            name: name.to_string(),
350            help: help.to_string(),
351            kind: MetricKind::Gauge,
352        });
353        gauge
354    }
355
356    /// 注册 Histogram
357    pub fn register_histogram(&self, name: &str, help: &str, buckets: Vec<f64>) -> Arc<Histogram> {
358        let mut histograms = self.histograms.write();
359        if let Some(h) = histograms.get(name) {
360            return h.clone();
361        }
362        // 最后一个 bucket 必须是 +Inf
363        let mut all_buckets = buckets;
364        all_buckets.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
365        if !all_buckets.contains(&f64::INFINITY) {
366            all_buckets.push(f64::INFINITY);
367        }
368        let count = all_buckets.len();
369        let histogram = Arc::new(Histogram {
370            name: name.to_string(),
371            buckets: all_buckets,
372            counts: Arc::new(RwLock::new(vec![0; count])),
373            sum: Arc::new(RwLock::new(0.0)),
374            count: Arc::new(RwLock::new(0)),
375        });
376        histograms.insert(name.to_string(), histogram.clone());
377
378        let mut metas = self.metas.write();
379        metas.push(MetricMeta {
380            name: name.to_string(),
381            help: help.to_string(),
382            kind: MetricKind::Histogram,
383        });
384        histogram
385    }
386
387    /// 渲染所有指标为 Prometheus 文本格式
388    pub fn render(&self) -> String {
389        let mut output = String::new();
390
391        // 输出 HELP/TYPE 头
392        let metas = self.metas.read();
393        let mut seen = std::collections::HashSet::new();
394        for meta in metas.iter() {
395            if seen.contains(&meta.name) {
396                continue;
397            }
398            seen.insert(meta.name.clone());
399            output.push_str(&format!("# HELP {} {}\n", meta.name, meta.help));
400            let type_str = match meta.kind {
401                MetricKind::Counter => "counter",
402                MetricKind::Gauge => "gauge",
403                MetricKind::Histogram => "histogram",
404            };
405            output.push_str(&format!("# TYPE {} {}\n", meta.name, type_str));
406        }
407
408        // 输出 Counter 值
409        let counters = self.counters.read();
410        for c in counters.values() {
411            output.push_str(&c.render());
412        }
413
414        // 输出 Gauge 值
415        let gauges = self.gauges.read();
416        for g in gauges.values() {
417            output.push_str(&g.render());
418        }
419
420        // 输出 Histogram 值
421        let histograms = self.histograms.read();
422        for h in histograms.values() {
423            output.push_str(&h.render());
424        }
425
426        output
427    }
428}
429
430/// 启动 Prometheus metrics HTTP server
431///
432/// 在指定地址暴露 `/metrics` 端点，返回 Prometheus 文本格式的指标数据。
433/// 每个连接在独立 tokio task 中处理。
434pub async fn start_metrics_server(
435    registry: Arc<MetricsRegistry>,
436    addr: std::net::SocketAddr,
437) -> Result<(), std::io::Error> {
438    use tokio::io::AsyncWriteExt;
439
440    let listener = tokio::net::TcpListener::bind(addr).await?;
441    loop {
442        let (mut stream, _) = listener.accept().await?;
443        let registry = registry.clone();
444        tokio::spawn(async move {
445            let metrics = registry.render();
446            let response = format!(
447                "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\n\r\n{}",
448                metrics.len(),
449                metrics
450            );
451            let _ = stream.write_all(response.as_bytes()).await;
452        });
453    }
454}
455
456/// v3.8.0：metrics 端点访问控制配置
457#[cfg(feature = "prod-metrics-acl")]
458#[derive(Debug, Clone)]
459pub struct MetricsAccessControl {
460    /// 是否启用访问控制
461    pub enabled: bool,
462    /// IP 白名单（CIDR 格式字符串，如 "10.0.0.0/8"）
463    pub ip_whitelist: Vec<String>,
464    /// Bearer Token 鉴权
465    pub bearer_token: Option<String>,
466    /// Basic Auth 鉴权 (username, password)
467    pub basic_auth: Option<(String, String)>,
468}
469
470#[cfg(feature = "prod-metrics-acl")]
471impl MetricsAccessControl {
472    /// 创建禁用访问控制的配置
473    pub fn disabled() -> Self {
474        Self {
475            enabled: false,
476            ip_whitelist: Vec::new(),
477            bearer_token: None,
478            basic_auth: None,
479        }
480    }
481
482    /// 检查 IP 是否在白名单中
483    pub fn check_ip_whitelist(peer_ip: &str, whitelist: &[String]) -> bool {
484        if whitelist.is_empty() {
485            return true;
486        }
487        for cidr in whitelist {
488            if ip_in_cidr(peer_ip, cidr) {
489                return true;
490            }
491        }
492        false
493    }
494
495    /// 检查 Bearer Token（常量时间比较）
496    pub fn check_bearer_token(auth_header: Option<&str>, expected: &str) -> bool {
497        use subtle::ConstantTimeEq;
498        if let Some(header) = auth_header {
499            if let Some(token) = header.strip_prefix("Bearer ") {
500                return token.as_bytes().ct_eq(expected.as_bytes()).into();
501            }
502        }
503        false
504    }
505
506    /// 检查 Basic Auth（常量时间比较）
507    pub fn check_basic_auth(
508        auth_header: Option<&str>,
509        expected_user: &str,
510        expected_pass: &str,
511    ) -> bool {
512        use subtle::ConstantTimeEq;
513        if let Some(header) = auth_header {
514            if let Some(encoded) = header.strip_prefix("Basic ") {
515                if let Ok(decoded) = base64_decode(encoded) {
516                    let parts: Vec<&str> = decoded.splitn(2, ':').collect();
517                    if parts.len() == 2 {
518                        return parts[0].as_bytes().ct_eq(expected_user.as_bytes()).into()
519                            && parts[1].as_bytes().ct_eq(expected_pass.as_bytes()).into();
520                    }
521                }
522            }
523        }
524        false
525    }
526
527    /// 综合鉴权检查
528    pub fn check_access(&self, peer_ip: &str, auth_header: Option<&str>) -> bool {
529        if !self.enabled {
530            return true;
531        }
532        if !Self::check_ip_whitelist(peer_ip, &self.ip_whitelist) {
533            return false;
534        }
535        if let Some(ref expected_token) = self.bearer_token {
536            if !Self::check_bearer_token(auth_header, expected_token) {
537                return false;
538            }
539        }
540        if let Some((ref user, ref pass)) = self.basic_auth {
541            if !Self::check_basic_auth(auth_header, user, pass) {
542                return false;
543            }
544        }
545        true
546    }
547}
548
549#[cfg(feature = "prod-metrics-acl")]
550fn ip_in_cidr(ip: &str, cidr: &str) -> bool {
551    if let Some(slash_pos) = cidr.find('/') {
552        let network = &cidr[..slash_pos];
553        let prefix_len: u32 = cidr[slash_pos + 1..].parse().unwrap_or(0);
554        if let (Ok(ip_addr), Ok(net_addr)) = (
555            ip.parse::<std::net::IpAddr>(),
556            network.parse::<std::net::IpAddr>(),
557        ) {
558            match (ip_addr, net_addr) {
559                (std::net::IpAddr::V4(ip4), std::net::IpAddr::V4(net4)) => {
560                    if prefix_len > 32 {
561                        return false;
562                    }
563                    let mask = if prefix_len == 0 {
564                        0u32
565                    } else {
566                        (!0u32) << (32 - prefix_len)
567                    };
568                    let ip_int = u32::from(ip4);
569                    let net_int = u32::from(net4);
570                    return (ip_int & mask) == (net_int & mask);
571                }
572                (std::net::IpAddr::V6(_), std::net::IpAddr::V6(_)) => {
573                    return ip == network;
574                }
575                _ => return false,
576            }
577        }
578    }
579    ip == cidr
580}
581
582#[cfg(feature = "prod-metrics-acl")]
583fn base64_decode(input: &str) -> Result<String, String> {
584    use base64::engine::general_purpose::STANDARD;
585    use base64::Engine;
586    let bytes = STANDARD.decode(input).map_err(|e| e.to_string())?;
587    String::from_utf8(bytes).map_err(|e| e.to_string())
588}
589
590/// v3.8.0：带访问控制的 metrics 端点
591#[cfg(feature = "prod-metrics-acl")]
592pub async fn start_metrics_server_with_acl(
593    registry: Arc<MetricsRegistry>,
594    addr: std::net::SocketAddr,
595    acl: MetricsAccessControl,
596) -> Result<(), std::io::Error> {
597    use tokio::io::AsyncWriteExt;
598
599    let listener = tokio::net::TcpListener::bind(addr).await?;
600    loop {
601        let (mut stream, peer) = listener.accept().await?;
602        let registry = registry.clone();
603        let acl = acl.clone();
604        tokio::spawn(async move {
605            let peer_ip = peer.ip().to_string();
606            let metrics = registry.render();
607            let auth_header = None;
608            if !acl.check_access(&peer_ip, auth_header) {
609                let response = "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n";
610                let _ = stream.write_all(response.as_bytes()).await;
611                return;
612            }
613            let response = format!(
614                "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\n\r\n{}",
615                metrics.len(),
616                metrics
617            );
618            let _ = stream.write_all(response.as_bytes()).await;
619        });
620    }
621}
622
623#[cfg(test)]
624mod tests {
625    use super::*;
626
627    #[test]
628    fn test_counter_basic() {
629        let registry = MetricsRegistry::new();
630        let counter = registry.register_counter("test_counter", "Test counter");
631        counter.inc();
632        counter.inc_by(2.5);
633        assert_eq!(counter.value(), 3.5);
634    }
635
636    #[test]
637    fn test_gauge_basic() {
638        let registry = MetricsRegistry::new();
639        let gauge = registry.register_gauge("test_gauge", "Test gauge");
640        gauge.set(10.0);
641        gauge.inc();
642        gauge.dec_by(3.0);
643        assert_eq!(gauge.value(), 8.0);
644    }
645
646    #[test]
647    fn test_histogram_basic() {
648        let registry = MetricsRegistry::new();
649        let histogram =
650            registry.register_histogram("test_histogram", "Test histogram", vec![0.1, 0.5, 1.0]);
651        histogram.observe(0.05);
652        histogram.observe(0.2);
653        histogram.observe(0.6);
654        histogram.observe(1.5);
655
656        assert_eq!(histogram.count(), 4);
657        assert!((histogram.sum() - 2.35).abs() < 1e-9);
658    }
659
660    #[test]
661    fn test_render_prometheus_format() {
662        let registry = MetricsRegistry::new();
663        let counter = registry.register_counter("ops_total", "Total operations");
664        let gauge = registry.register_gauge("conn_active", "Active connections");
665        let histogram =
666            registry.register_histogram("latency_seconds", "Latency in seconds", vec![0.01, 0.1]);
667
668        counter.inc_by(10.0);
669        gauge.set(5.0);
670        histogram.observe(0.005);
671        histogram.observe(0.05);
672        histogram.observe(0.5);
673
674        let output = registry.render();
675        assert!(output.contains("# HELP ops_total Total operations"));
676        assert!(output.contains("# TYPE ops_total counter"));
677        assert!(output.contains("ops_total 10"));
678        assert!(output.contains("conn_active 5"));
679        assert!(output.contains("latency_seconds_bucket{le=\"0.01\"} 1"));
680        assert!(output.contains("latency_seconds_bucket{le=\"0.1\"} 2"));
681        assert!(output.contains("latency_seconds_sum 0.555"));
682        assert!(output.contains("latency_seconds_count 3"));
683    }
684
685    #[test]
686    fn test_counter_with_labels() {
687        let registry = MetricsRegistry::new();
688        let mut labels = HashMap::new();
689        labels.insert("method".to_string(), "GET".to_string());
690        labels.insert("status".to_string(), "200".to_string());
691
692        let counter =
693            registry.register_counter_with_labels("http_requests_total", "HTTP requests", labels);
694        counter.inc();
695        let output = registry.render();
696        // HashMap 顺序未定义，分别验证各标签
697        assert!(output.contains("http_requests_total{"));
698        assert!(output.contains("method=\"GET\""));
699        assert!(output.contains("status=\"200\""));
700        assert!(output.contains("} 1"));
701    }
702
703    #[cfg(feature = "prod-metrics-acl")]
704    mod prod_metrics_acl_tests {
705        use super::*;
706
707        #[test]
708        fn test_ip_whitelist_match() {
709            let whitelist = vec!["10.0.0.0/8".to_string()];
710            assert!(MetricsAccessControl::check_ip_whitelist(
711                "10.1.2.3", &whitelist
712            ));
713            assert!(!MetricsAccessControl::check_ip_whitelist(
714                "192.168.1.1",
715                &whitelist
716            ));
717        }
718
719        #[test]
720        fn test_ip_whitelist_empty_allows_all() {
721            assert!(MetricsAccessControl::check_ip_whitelist("1.2.3.4", &[]));
722        }
723
724        #[test]
725        fn test_bearer_token_valid() {
726            assert!(MetricsAccessControl::check_bearer_token(
727                Some("Bearer secret123"),
728                "secret123"
729            ));
730        }
731
732        #[test]
733        fn test_bearer_token_invalid() {
734            assert!(!MetricsAccessControl::check_bearer_token(
735                Some("Bearer wrong"),
736                "secret123"
737            ));
738            assert!(!MetricsAccessControl::check_bearer_token(None, "secret123"));
739        }
740
741        #[test]
742        fn test_check_access_disabled_allows_all() {
743            let acl = MetricsAccessControl::disabled();
744            assert!(acl.check_access("1.2.3.4", None));
745        }
746
747        #[test]
748        fn test_check_access_enabled_ip_rejected() {
749            let acl = MetricsAccessControl {
750                enabled: true,
751                ip_whitelist: vec!["10.0.0.0/8".to_string()],
752                bearer_token: None,
753                basic_auth: None,
754            };
755            assert!(!acl.check_access("192.168.1.1", None));
756            assert!(acl.check_access("10.1.2.3", None));
757        }
758    }
759}
sz_orm_observability/lib.rs

sz_orm_observability/
lib.rs
