endpoint-libs 1.5.1

Common dependencies to be used with Pathscale projects, projects that use [endpoint-gen](https://github.com/pathscale/endpoint-gen), and projects that use honey_id-types.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
//! Tracing-based logging setup with stdout, file, and OpenTelemetry logging support

use std::{path::PathBuf, time::Duration};

use chrono::SecondsFormat;
use eyre::{DefaultHandler, EyreHandler, bail};
use tracing::Subscriber;
use tracing_appender::rolling::RollingFileAppender;
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::{
    EnvFilter, Layer, Registry,
    layer::SubscriberExt,
    registry,
    reload::{self, Handle},
};

#[cfg(feature = "log_throttling")]
use tracing_throttle::TracingRateLimitLayer;

#[cfg(feature = "error_aggregation")]
use {crate::libs::log::error_aggregation::*, std::sync::Arc};

#[deprecated(
    since = "1.3.0",
    note = "This code is not used in current projects and should probably not be used going forward"
)]
pub mod legacy;

#[cfg(feature = "error_aggregation")]
pub mod error_aggregation;
pub mod level_filter;
pub mod otel;

pub use level_filter::*;
pub use otel::{OtelConfig, OtelGuards, OtelProtocol};

// Public re-export of Rotation so clients don't need to include tracing_appender just for log setup
pub use tracing_appender::rolling::Rotation as LogRotation;

pub use tracing_appender::non_blocking::WorkerGuard;

#[derive(Debug)]
pub struct LoggingConfig {
    pub level: LogLevel,
    pub file_config: Option<FileLoggingConfig>,
    /// OpenTelemetry configuration for log/trace forwarding to an OTLP collector.
    /// By default, OTel is disabled. Set `enabled: true` and configure the endpoint
    /// to forward traces and logs to an OTel collector.
    pub otel_config: OtelConfig,
    #[cfg(feature = "error_aggregation")]
    pub error_aggregation: ErrorAggregationConfig,
    #[cfg(feature = "log_throttling")]
    pub throttling_config: Option<LogThrottlingConfig>,
}

#[derive(Debug, Clone)]
pub struct FileLoggingConfig {
    pub path: PathBuf,
    /// The name of the log file before the '.log', e.g. <file_prefix>.log will be the final log file.
    /// Set to None for the current timestamp
    pub file_prefix: Option<String>,
    // Specifying None means that there will be one log file per program execution
    pub rotation: Option<LogRotation>,
}

#[derive(Debug, Default, Clone)]
pub struct LogThrottlingConfig {
    /// How often to emit throttling summaries as WARN events, set to None to disable entirely
    pub summary_emission_interval: Option<Duration>,
    /// How often to throttling metrics as INFO events, set to None to disable entirely
    pub metrics_emission_interval: Option<Duration>,
    /// Fields to exclude from uniqueness checks, set to None to disable entirely.<br><br>
    /// Example:
    /// ```ignore
    /// tracing::info!(user_id=1, "User joined");
    /// tracing::info!(user_id=2, "User joined");  
    ///```
    /// If the `user_id` field is excluded, these will be treated as exactly the same log, so multiple users join messages could be throttled
    pub excluded_fields: Option<Vec<String>>,
    /// Targets to exempt from any throttling. This allows the caller to ensure that any high priority logs are always displayed.<br><br>
    /// Example:
    /// ```ignore
    /// let exemptions = Some(vec!["nothrottle"]); // Assuming this is passed during config stage
    ///
    /// tracing::error!(target: "nothrottle", user_id=1, "User joined"); // Will never be throttled
    /// tracing::error!(user_id=2, "User joined");  // Can possibly be throttled
    /// ```
    pub exemptions: Option<Vec<String>>,
}

pub struct LogSetupReturn {
    pub reload_handle: LogReloadHandle,
    pub log_guards: (WorkerGuard, Option<WorkerGuard>),
    /// OpenTelemetry guards (tracer + logger providers). Must be kept alive to ensure 
    /// pending traces and logs are flushed to the OTLP collector on shutdown.
    pub otel_guards: Option<OtelGuards>,
    #[cfg(feature = "error_aggregation")]
    pub errors_container: Arc<ErrorAggregationContainer>,
    #[cfg(feature = "log_throttling")]
    /// Call shutdown on this during graceful shutdown
    pub log_throttling_handle: TracingRateLimitLayer,
}

/// Internal struct to hold the result of building the logging subscriber
struct LoggingSubscriberParts {
    subscriber: Box<dyn Subscriber + Send + Sync + 'static>,
    reload_handle: LogReloadHandle,
    log_guards: (WorkerGuard, Option<WorkerGuard>), // Stdout and optional file log guards
    /// OpenTelemetry guards (tracer + logger providers).
    otel_guards: Option<OtelGuards>,
    #[cfg(feature = "error_aggregation")]
    errors_container: Arc<ErrorAggregationContainer>,
    #[cfg(feature = "log_throttling")]
    log_throttling_handle: TracingRateLimitLayer,
}

/// Internal function that builds the logging subscriber without initializing it.
/// Returns the subscriber along with reload handles and guards that need to be retained.
fn build_logging_subscriber(config: LoggingConfig) -> eyre::Result<LoggingSubscriberParts> {
    // Build global env filter and wrap in reload::Layer for runtime reloading
    let global_env_filter = build_env_filter(config.level)?;
    let (reloadable_global_filter, global_reload_handle) = reload::Layer::new(global_env_filter);

    let (non_blocking_stdout, stdout_guard) = tracing_appender::non_blocking(std::io::stdout());

    let stdout_layer = tracing_subscriber::fmt::layer()
        .with_thread_names(true)
        .with_line_number(true)
        .with_writer(non_blocking_stdout);

    #[cfg(feature = "error_aggregation")]
    let error_aggregation_config = config.error_aggregation.clone();

    #[cfg(feature = "log_throttling")]
    let throttling_config = config.throttling_config.clone().unwrap_or_default();

    #[cfg(feature = "log_throttling")]
    let mut exemptions = vec!["tracing_throttle::infrastructure::layer".to_string()]; // It seems like it tries to throttle its own logs
    #[cfg(feature = "log_throttling")]
    exemptions.extend(throttling_config.exemptions.unwrap_or_default());

    #[cfg(feature = "log_throttling")]
    let rate_limit_filter = TracingRateLimitLayer::builder()
        .with_excluded_fields(throttling_config.excluded_fields.unwrap_or_default())
        .with_exempt_targets(exemptions)
        .with_active_emission(throttling_config.summary_emission_interval.is_some())
        .with_summary_interval(
            throttling_config
                .summary_emission_interval
                .unwrap_or(Duration::from_secs(5 * 60)),
        )
        .build()
        .expect("Error building tracing rate limit layer");

    #[cfg(feature = "log_throttling")]
    if let Some(metrics_duration) = throttling_config.metrics_emission_interval {
        let metrics = rate_limit_filter.metrics().clone();

        // Periodic metrics reporting
        tokio::spawn(async move {
            loop {
                tokio::time::sleep(metrics_duration).await;

                let snapshot = metrics.snapshot();
                tracing::info!(
                    events_allowed = snapshot.events_allowed,
                    events_suppressed = snapshot.events_suppressed,
                    suppression_rate = format!("{:.1}%", snapshot.suppression_rate() * 100.0),
                    "Rate limiting metrics"
                );
            }
        });
    }

    #[cfg(feature = "log_throttling")]
    let log_throttling_handle = rate_limit_filter.clone();

    let (file_layer, file_guard) = match config.file_config {
        None => (None, None),
        Some(file_config) => {
            let appender = RollingFileAppender::builder()
                .rotation(file_config.rotation.unwrap_or(LogRotation::NEVER))
                .filename_prefix(file_config.file_prefix.unwrap_or_else(|| {
                    chrono::Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true)
                })) // Default filename prefix is a timestamp. RollingFileAppender requires a prefix if we are using a simple suffix like ".log"
                .filename_suffix("log")
                .build(file_config.path)?;

            let (non_blocking_appender, guard) = tracing_appender::non_blocking(appender);

            let base_layer = tracing_subscriber::fmt::layer()
                .with_thread_names(true)
                .with_line_number(true)
                .with_ansi(false)
                .with_writer(non_blocking_appender);

            #[cfg(feature = "log_throttling")]
            let file_layer = base_layer.with_filter(rate_limit_filter);

            #[cfg(not(feature = "log_throttling"))]
            let file_layer = base_layer;

            (Some(file_layer), Some(guard))
        }
    };

    let reload_handle = LogReloadHandle(global_reload_handle);

    // Build OTel layer (separate, parallel to stdout/file layers)
    let otel_result = otel::build_otel_layer(&config.otel_config);
    let otel_guards = otel_result.guards;
    let otel_tracer = otel_result.tracer;

    // --- Subscriber Composition ---
    
    // Start with Registry and Global Filter
    let subscriber = registry().with(reloadable_global_filter);

    // Compose the "Sink" layers (Stdout + File)
    let sinks = stdout_layer.and_then(file_layer);

    // Add Error Aggregation if enabled
    #[cfg(feature = "error_aggregation")]
    let (sinks, errors_container) = {
        use crate::libs::log::error_aggregation::get_error_aggregation;
        let (container, error_layer) = get_error_aggregation(error_aggregation_config);
        (sinks.and_then(error_layer), container)
    };

    // Combine Subscriber with Sinks
    let subscriber = subscriber.with(sinks);

    // Add OTel Traces and Logs layers if enabled
    let subscriber: Box<dyn Subscriber + Send + Sync + 'static> = match (otel_tracer, &otel_guards) {
        (Some(tracer), Some(guards)) => {
            let trace_layer = OpenTelemetryLayer::new(tracer);
            let log_layer = opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&guards.logger_provider);
            Box::new(subscriber.with(trace_layer).with(log_layer))
        }
        _ => Box::new(subscriber),
    };

    Ok(LoggingSubscriberParts {
        subscriber,
        reload_handle,
        log_guards: (stdout_guard, file_guard),
        otel_guards,
        #[cfg(feature = "error_aggregation")]
        errors_container,
        #[cfg(feature = "log_throttling")]
        log_throttling_handle,
    })
}

/// Sets up a fully filtered tracing logging system with a global env filter applied at the registry level.
/// This ensures all layers (stdout, file, etc.) only receive events that pass the global filter.
///
/// By default with the minimal config a stdout layer is setup with a custom env filter that filters according to the specified log level,
/// as well as according to a list of hardcoded crates. See [build_env_filter] for hardcoded crate log filtering.
/// Optionally, file logging can be configured with [FileLoggingConfig], which minimally will create a timestamp prefixed log file in the given
/// path that does not rotate (one log file per program execution)
///
/// Extra configuration for file logging is:
/// - Log file rotation. See [LogRotation] for possible options
///
/// Returns [LogSetupReturn], which is a composite struct containing objects that need to be retained by the client such as:
/// - [LogReloadHandle], for setting a new global log level during runtime
/// - [WorkerGuard], so that the non-blocking file writer can continue writing. This cannot be dropped and needs to be kept alive for the duration of the program execution
/// - [Option<OtelGuards>], the OTel guards. Must be kept alive to flush pending traces and logs on shutdown.
/// - [ErrorAggregationContainer], if the [error_aggregation] feature is enabled. This object allows recent errors to be queried from the logging framework
pub fn setup_logging(config: LoggingConfig) -> eyre::Result<LogSetupReturn> {
    use tracing_subscriber::util::SubscriberInitExt;

    let parts = build_logging_subscriber(config)?;

    parts.subscriber.init();

    Ok(LogSetupReturn {
        reload_handle: parts.reload_handle,
        log_guards: parts.log_guards,
        otel_guards: parts.otel_guards,
        #[cfg(feature = "error_aggregation")]
        errors_container: parts.errors_container,
        #[cfg(feature = "log_throttling")]
        log_throttling_handle: parts.log_throttling_handle,
    })
}

/// Handle for reloading the global EnvFilter at runtime
pub type GlobalLogReloadHandle = Handle<EnvFilter, Registry>;

/// Handle for reloading the global log level at runtime
#[derive(Clone)]
pub struct LogReloadHandle(GlobalLogReloadHandle);

impl std::fmt::Debug for LogReloadHandle {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("LogReloadHandle")
            .field("inner", &"Handle<EnvFilter, Registry>")
            .finish()
    }
}

impl LogReloadHandle {
    /// Sets a new global log level that applies to all layers (stdout, file, etc.)
    /// Errors will be logged by this function and can be safely ignored, but are returned for display purposes
    pub fn set_log_level(&self, level: LogLevel) -> eyre::Result<()> {
        match build_env_filter(level) {
            Ok(filter) => match self.0.modify(|env_filter| *env_filter = filter) {
                Ok(_) => Ok(()),
                Err(error) => {
                    tracing::error!(
                        ?error,
                        "Error setting new global log filter. Ignoring reload attempt"
                    );
                    bail!("Error setting new global log filter: {error}. Ignoring reload attempt")
                }
            },
            Err(error) => {
                tracing::error!(
                    ?error,
                    "Error building new filter from given log level. Ignoring reload attempt"
                );
                bail!(
                    "Error building new filter from given log level: {error}. Ignoring reload attempt"
                )
            }
        }
    }
}

/// This is used to wrap the default EyreHandler but simply expose the [CustomEyreHandler::location] field via [CustomEyreHandler::get_location]
/// This allows the code in [crate::libs::ws::internal_error_to_resp] to access the original caller location which is then stored in a [tracing::Field]
/// for access within [error_aggregation::ErrorAggregationLayer::on_event] (see trait impl) so that the original caller can be recorded within the displayed target
pub struct CustomEyreHandler {
    default_handler: Box<dyn EyreHandler>,
    location: Option<&'static std::panic::Location<'static>>,
}

impl CustomEyreHandler {
    pub fn default_with_location_saving(
        error: &(dyn std::error::Error + 'static),
    ) -> Box<dyn EyreHandler> {
        Box::new(Self {
            default_handler: DefaultHandler::default_with(error),
            location: None,
        })
    }

    pub fn get_location(&self) -> &Option<&'static std::panic::Location<'static>> {
        &self.location
    }
}

impl EyreHandler for CustomEyreHandler {
    fn display(
        &self,
        error: &(dyn std::error::Error + 'static),
        f: &mut core::fmt::Formatter<'_>,
    ) -> core::fmt::Result {
        self.default_handler.display(error, f)
    }

    fn debug(
        &self,
        error: &(dyn std::error::Error + 'static),
        f: &mut core::fmt::Formatter<'_>,
    ) -> core::fmt::Result {
        self.default_handler.debug(error, f)
    }

    fn track_caller(&mut self, location: &'static std::panic::Location<'static>) {
        self.location = Some(location); // Store the location for access later
        self.default_handler.track_caller(location);
    }
}

#[cfg(test)]
pub struct LogSetupReturnTest {
    _guard: tracing::subscriber::DefaultGuard,
    reload_handle: LogReloadHandle,
    #[allow(dead_code)]
    log_guards: (WorkerGuard, Option<WorkerGuard>),
    #[allow(dead_code)]
    otel_guards: Option<OtelGuards>,
    #[cfg(feature = "error_aggregation")]
    #[allow(dead_code)]
    errors_container: Arc<ErrorAggregationContainer>,
}

/// Test-specific logging setup that uses thread-local scoped dispatcher instead of global
/// This allows multiple tests to run without conflicting over the global subscriber
#[cfg(test)]
pub fn setup_logging_test(config: LoggingConfig) -> eyre::Result<LogSetupReturnTest> {
    let parts = build_logging_subscriber(config)?;

    // Use thread-local default instead of global
    let guard = tracing::subscriber::set_default(parts.subscriber);

    Ok(LogSetupReturnTest {
        _guard: guard,
        reload_handle: parts.reload_handle,
        log_guards: parts.log_guards,
        otel_guards: parts.otel_guards,
        #[cfg(feature = "error_aggregation")]
        errors_container: parts.errors_container,
    })
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use tracing::{debug, error, info};

    #[cfg(feature = "error_aggregation")]
    fn default_error_aggregation_config() -> ErrorAggregationConfig {
        ErrorAggregationConfig {
            limit: 100,
            normalize: true,
        }
    }

    /// Basic test that verifies logging setup succeeds and logs can be emitted
    #[tokio::test]
    async fn test_basic_logging_stdout() {
        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: None,
            otel_config: OtelConfig::default(),
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: Some(LogThrottlingConfig::default()),
        };

        let _guard = setup_logging_test(config);
        assert!(_guard.is_ok(), "Failed to setup logging");

        // Emit some logs to verify the system works
        info!("Test info message");
        error!("Test error message");
    }

    /// Test file logging functionality including:
    /// - Basic file logging
    /// - Log level filtering (separate stdout vs file levels)
    /// - Log level reloading at runtime
    #[tokio::test]
    async fn test_file_logging_comprehensive() {
        let temp_dir = tempfile::tempdir().unwrap();

        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: Some(FileLoggingConfig {
                path: temp_dir.path().to_path_buf(),
                file_prefix: Some("test".to_string()),
                rotation: None,
            }),
            otel_config: OtelConfig::default(),
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: Some(LogThrottlingConfig::default()),
        };

        let log_setup = setup_logging_test(config).unwrap();

        // Test 1: Basic logging and level filtering (before reload)
        info!("info_message_before_reload");
        debug!("debug_message_before_reload");
        error!("error_message");

        std::thread::sleep(std::time::Duration::from_millis(100));

        // Test 2: Reload global log level to DEBUG
        let result = log_setup.reload_handle.set_log_level(LogLevel::Debug);
        assert!(result.is_ok());

        // Test 3: Verify debug logs now appear after reload
        info!("info_message_after_reload");
        debug!("debug_message_after_reload");

        std::thread::sleep(std::time::Duration::from_millis(100));
        drop(log_setup);

        std::thread::sleep(std::time::Duration::from_millis(100));

        // Verify file was created and contains expected logs
        let log_files: Vec<_> = fs::read_dir(temp_dir.path()).unwrap().collect();
        assert_eq!(log_files.len(), 1, "Expected exactly one log file");

        let log_file = log_files[0].as_ref().unwrap();
        let log_contents = fs::read_to_string(log_file.path()).unwrap();

        // Verify messages before reload
        assert!(log_contents.contains("info_message_before_reload"));
        assert!(
            !log_contents.contains("debug_message_before_reload"),
            "Debug should not appear before reload"
        );
        assert!(log_contents.contains("error_message"));

        // Verify messages after reload
        assert!(log_contents.contains("info_message_after_reload"));

        // Global log level reload applies to all layers including file
        assert!(
            log_contents.contains("debug_message_after_reload"),
            "Debug should appear after reload to DEBUG level"
        );
    }

    // === OTel Integration Tests ===

    /// Test that setup succeeds when OTel is disabled (default config)
    /// and otel_guards is None
    #[tokio::test]
    async fn test_otel_disabled_returns_none_guards() {
        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: None,
            otel_config: OtelConfig::default(), // enabled: false
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: None,
        };

        let result = setup_logging_test(config);
        assert!(result.is_ok(), "Setup should succeed with OTel disabled");

        let guard = result.unwrap();
        assert!(
            guard.otel_guards.is_none(),
            "otel_guards should be None when OTel is disabled"
        );
    }

    /// Test that OTel initialization succeeds even with an unreachable endpoint
    /// The SDK initializes asynchronously, so guards ARE present even if the endpoint
    /// is unreachable. Export failures happen at runtime, not at setup time.
    #[tokio::test]
    async fn test_otel_graceful_degradation_unreachable_endpoint() {
        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: None,
            otel_config: OtelConfig {
                enabled: true,
                endpoint: Some("http://nonexistent.invalid:4317".to_string()),
                ..OtelConfig::default()
            },
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: None,
        };

        // Setup should succeed - the SDK doesn't validate connectivity at init time
        let result = setup_logging_test(config);
        assert!(result.is_ok(), "Setup should succeed even with unreachable OTel endpoint");

        let guard = result.unwrap();
        // Guards ARE present because the SDK creates them synchronously.
        // Export failures happen asynchronously when spans/logs are batched.
        // The key assertion is that setup() didn't return an error.
        assert!(
            guard.otel_guards.is_some(),
            "otel_guards should be Some even with unreachable endpoint (SDK initializes synchronously)"
        );
    }

    /// Test that file logging works alongside OTel enabled (but with invalid endpoint)
    /// This verifies that the layer composition doesn't break when OTel is attempted
    #[tokio::test]
    async fn test_file_logging_with_otel_attempted() {
        let temp_dir = tempfile::tempdir().unwrap();

        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: Some(FileLoggingConfig {
                path: temp_dir.path().to_path_buf(),
                file_prefix: Some("otel_test".to_string()),
                rotation: None,
            }),
            otel_config: OtelConfig {
                enabled: true,
                endpoint: Some("http://localhost:4317".to_string()), // Not running
                ..OtelConfig::default()
            },
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: None,
        };

        let log_setup = setup_logging_test(config).unwrap();

        // Emit logs - these should go to file regardless of OTel status
        info!("file_log_with_otel_attempted");
        error!("error_log_with_otel_attempted");

        std::thread::sleep(std::time::Duration::from_millis(100));
        drop(log_setup);
        std::thread::sleep(std::time::Duration::from_millis(100));

        // Verify file was created and contains logs
        let log_files: Vec<_> = fs::read_dir(temp_dir.path()).unwrap().collect();
        assert_eq!(log_files.len(), 1, "Expected exactly one log file");

        let log_contents = fs::read_to_string(log_files[0].as_ref().unwrap().path()).unwrap();
        assert!(log_contents.contains("file_log_with_otel_attempted"));
        assert!(log_contents.contains("error_log_with_otel_attempted"));
    }

    /// Test that log level reload works correctly when OTel is enabled
    #[tokio::test]
    async fn test_log_level_reload_with_otel_enabled() {
        let temp_dir = tempfile::tempdir().unwrap();

        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: Some(FileLoggingConfig {
                path: temp_dir.path().to_path_buf(),
                file_prefix: Some("reload_test".to_string()),
                rotation: None,
            }),
            otel_config: OtelConfig {
                enabled: true,
                endpoint: Some("http://localhost:4317".to_string()),
                ..OtelConfig::default()
            },
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: None,
        };

        let log_setup = setup_logging_test(config).unwrap();

        // Log at INFO level (should appear)
        info!("info_before_reload");
        debug!("debug_before_reload"); // Should NOT appear

        std::thread::sleep(std::time::Duration::from_millis(100));

        // Reload to DEBUG
        let result = log_setup.reload_handle.set_log_level(LogLevel::Debug);
        assert!(result.is_ok(), "Log level reload should succeed with OTel enabled");

        // Log at DEBUG level (should now appear)
        info!("info_after_reload");
        debug!("debug_after_reload");

        std::thread::sleep(std::time::Duration::from_millis(100));
        drop(log_setup);
        std::thread::sleep(std::time::Duration::from_millis(100));

        let log_files: Vec<_> = fs::read_dir(temp_dir.path()).unwrap().collect();
        let log_contents = fs::read_to_string(log_files[0].as_ref().unwrap().path()).unwrap();

        assert!(log_contents.contains("info_before_reload"));
        assert!(!log_contents.contains("debug_before_reload"), "Debug should not appear before reload");
        assert!(log_contents.contains("info_after_reload"));
        assert!(log_contents.contains("debug_after_reload"), "Debug should appear after reload");
    }

    /// Test that OTel config fields are correctly structured
    #[tokio::test]
    async fn test_otel_config_fields() {
        use std::collections::HashMap;

        let config = OtelConfig {
            enabled: true,
            service_name: Some("test-service".to_string()),
            endpoint: Some("http://collector:4317".to_string()),
            protocol: otel::OtelProtocol::HttpProtobuf,
            headers: {
                let mut h = HashMap::new();
                h.insert("x-api-key".to_string(), "secret-key".to_string());
                h
            },
        };

        assert!(config.enabled);
        assert_eq!(config.service_name, Some("test-service".to_string()));
        assert_eq!(config.endpoint, Some("http://collector:4317".to_string()));
        assert!(matches!(config.protocol, otel::OtelProtocol::HttpProtobuf));
        assert_eq!(config.headers.get("x-api-key"), Some(&"secret-key".to_string()));

        // Default config should be disabled
        let default_config = OtelConfig::default();
        assert!(!default_config.enabled);
        assert!(default_config.service_name.is_none());
        assert!(default_config.endpoint.is_none());
        assert!(matches!(default_config.protocol, otel::OtelProtocol::Grpc));
        assert!(default_config.headers.is_empty());
    }

    /// Test that setup succeeds with OTel enabled but no endpoint specified
    /// (should use env var fallback or SDK defaults)
    #[tokio::test]
    async fn test_otel_enabled_no_endpoint_uses_fallback() {
        let config = LoggingConfig {
            level: LogLevel::Info,
            file_config: None,
            otel_config: OtelConfig {
                enabled: true,
                endpoint: None, // No endpoint - should use SDK defaults
                ..OtelConfig::default()
            },
            #[cfg(feature = "error_aggregation")]
            error_aggregation: default_error_aggregation_config(),
            #[cfg(feature = "log_throttling")]
            throttling_config: None,
        };

        // Setup should succeed (exporter will use SDK defaults)
        let result = setup_logging_test(config);
        assert!(result.is_ok(), "Setup should succeed with OTel enabled but no endpoint");

        let guard = result.unwrap();
        // Guards may or may not be present depending on whether SDK defaults work
        // The key thing is setup didn't panic or return an error
        let _ = guard.otel_guards;
    }
}