arqen 0.11.2

Backend infrastructure for agent-ready applications
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
//! Health and readiness module for Arqen.
//!
//! Provides dependency checks with timeouts, degraded states, parallel execution,
//! and HTTP endpoint integration for Kubernetes-style liveness/readiness probes.

use std::sync::Arc;
use std::time::{Duration, Instant};

use async_trait::async_trait;
use serde::{Deserialize, Serialize};

use crate::config::ThingdSyncMode;

/// Health status of a dependency.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum HealthStatus {
    /// Dependency is healthy.
    Healthy,
    /// Dependency is degraded but functional.
    Degraded { reason: String },
    /// Dependency is unhealthy.
    Unhealthy { reason: String },
}

impl HealthStatus {
    /// Check if status is healthy.
    pub fn is_healthy(&self) -> bool {
        matches!(self, HealthStatus::Healthy)
    }

    /// Check if status is degraded.
    pub fn is_degraded(&self) -> bool {
        matches!(self, HealthStatus::Degraded { .. })
    }

    /// Check if status is unhealthy.
    pub fn is_unhealthy(&self) -> bool {
        matches!(self, HealthStatus::Unhealthy { .. })
    }

    /// Convert to HTTP status code.
    pub fn to_http_status(&self) -> u16 {
        match self {
            HealthStatus::Healthy => 200,
            HealthStatus::Degraded { .. } => 200,
            HealthStatus::Unhealthy { .. } => 503,
        }
    }
}

impl std::fmt::Display for HealthStatus {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            HealthStatus::Healthy => write!(f, "healthy"),
            HealthStatus::Degraded { reason } => write!(f, "degraded: {}", reason),
            HealthStatus::Unhealthy { reason } => write!(f, "unhealthy: {}", reason),
        }
    }
}

/// Trait for health checks.
#[async_trait]
pub trait HealthCheck: Send + Sync {
    /// Name of the dependency.
    fn name(&self) -> &str;

    /// Perform the health check.
    async fn check(&self) -> HealthStatus;

    /// Timeout for the check.
    fn timeout(&self) -> Duration {
        Duration::from_secs(5)
    }

    /// Whether this check is required for readiness.
    fn required_for_readiness(&self) -> bool {
        true
    }
}

/// Reports whether the configured Thingd sync capability is available.
///
/// This is a capability check, not a network probe. Applications should
/// register a separate endpoint check when HTTP sync is enabled.
pub struct ThingdSyncHealth {
    mode: ThingdSyncMode,
    native_available: bool,
}

impl ThingdSyncHealth {
    #[must_use]
    pub fn new(mode: ThingdSyncMode) -> Self {
        Self {
            mode,
            native_available: false,
        }
    }

    /// Mark native sync healthy after its endpoint has been constructed.
    #[must_use]
    pub fn with_native_availability(mut self, available: bool) -> Self {
        self.native_available = available;
        self
    }
}

#[async_trait]
impl HealthCheck for ThingdSyncHealth {
    fn name(&self) -> &str {
        "thingd-sync-capability"
    }

    async fn check(&self) -> HealthStatus {
        match self.mode {
            ThingdSyncMode::Disabled | ThingdSyncMode::Http => HealthStatus::Healthy,
            ThingdSyncMode::Native if self.native_available => HealthStatus::Healthy,
            ThingdSyncMode::Native => HealthStatus::Degraded {
                reason: "native Thingd replication endpoint is unavailable".to_string(),
            },
        }
    }

    fn required_for_readiness(&self) -> bool {
        false
    }
}

/// Result of a single health check.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CheckResult {
    /// Name of the dependency.
    pub name: String,
    /// Status of the check.
    pub status: HealthStatus,
    /// Duration of the check in milliseconds.
    pub duration_ms: u64,
}

/// Overall health report.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthReport {
    /// Overall status.
    pub status: HealthStatus,
    /// Individual check results.
    pub checks: Vec<CheckResult>,
    /// Timestamp of the report.
    pub timestamp: String,
    /// Whether this is a liveness or readiness report.
    pub probe_type: ProbeType,
}

/// Type of health probe.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ProbeType {
    /// Liveness probe - is the application alive?
    Liveness,
    /// Readiness probe - is the application ready to serve traffic?
    Readiness,
}

/// Registry for health checks.
pub struct HealthRegistry {
    checks: Vec<Arc<dyn HealthCheck>>,
    default_timeout: Option<Duration>,
    startup_at: Instant,
    startup_delay: Duration,
}

impl HealthRegistry {
    /// Create a new health registry.
    pub fn new() -> Self {
        Self {
            checks: Vec::new(),
            default_timeout: None,
            startup_at: Instant::now(),
            startup_delay: Duration::ZERO,
        }
    }

    /// Override the timeout used for checks that do not provide an
    /// application-specific timeout.
    #[must_use]
    pub fn with_timeout(mut self, timeout: Duration) -> Self {
        self.default_timeout = Some(timeout);
        self
    }

    /// Keep readiness in startup grace until the delay has elapsed. Liveness
    /// remains available during this period so orchestrators do not restart a
    /// process merely because a dependency is still warming up.
    #[must_use]
    pub fn with_startup_delay(mut self, delay: Duration) -> Self {
        self.startup_at = Instant::now();
        self.startup_delay = delay;
        self
    }

    /// Apply runtime health settings from application configuration.
    pub fn configure(&mut self, timeout: Duration, startup_delay: Duration) {
        self.default_timeout = Some(timeout);
        self.startup_at = Instant::now();
        self.startup_delay = startup_delay;
    }

    /// Register a health check.
    pub fn register(&mut self, check: Arc<dyn HealthCheck>) {
        self.checks.push(check);
    }

    /// Run all health checks in parallel and return a report.
    pub async fn check_all(&self) -> HealthReport {
        self.check_with_type(ProbeType::Liveness).await
    }

    /// Run liveness checks (all checks).
    pub async fn check_liveness(&self) -> HealthReport {
        self.check_with_type(ProbeType::Liveness).await
    }

    /// Run readiness checks (only required checks).
    pub async fn check_readiness(&self) -> HealthReport {
        self.check_with_type(ProbeType::Readiness).await
    }

    /// Run checks with specified probe type.
    async fn check_with_type(&self, probe_type: ProbeType) -> HealthReport {
        if probe_type == ProbeType::Readiness && self.startup_at.elapsed() < self.startup_delay {
            let remaining = self.startup_delay.saturating_sub(self.startup_at.elapsed());
            tracing::debug!(
                remaining_ms = remaining.as_millis() as u64,
                "readiness is in startup grace period"
            );
            return HealthReport {
                status: HealthStatus::Unhealthy {
                    reason: format!(
                        "startup grace period active; {}ms remaining",
                        remaining.as_millis()
                    ),
                },
                checks: Vec::new(),
                timestamp: chrono::Utc::now().to_rfc3339(),
                probe_type,
            };
        }

        let checks_to_run: Vec<_> = match probe_type {
            ProbeType::Liveness => self.checks.clone(),
            ProbeType::Readiness => self
                .checks
                .iter()
                .filter(|c| c.required_for_readiness())
                .cloned()
                .collect(),
        };
        let default_timeout = self.default_timeout;

        let mut results = Vec::new();
        let mut overall_status = HealthStatus::Healthy;

        // Run checks in parallel
        let mut handles = Vec::new();
        for check in checks_to_run {
            handles.push(tokio::spawn(async move {
                let start = Instant::now();
                let check_timeout = check.timeout();
                let timeout = if check_timeout == Duration::from_secs(5) {
                    default_timeout.unwrap_or(check_timeout)
                } else {
                    check_timeout
                };
                let status = run_check_with_timeout(check.as_ref(), timeout).await;
                let duration_ms = start.elapsed().as_millis() as u64;
                if duration_ms > 3_000 {
                    tracing::warn!(check = %check.name(), duration_ms, "slow health check");
                }
                CheckResult {
                    name: check.name().to_string(),
                    status,
                    duration_ms,
                }
            }));
        }

        for handle in handles {
            if let Ok(result) = handle.await {
                // Update overall status
                match &result.status {
                    HealthStatus::Unhealthy { .. } => {
                        overall_status = result.status.clone();
                    }
                    HealthStatus::Degraded { .. } if overall_status.is_healthy() => {
                        overall_status = result.status.clone();
                    }
                    _ => {}
                }
                results.push(result);
            }
        }

        HealthReport {
            status: overall_status,
            checks: results,
            timestamp: chrono::Utc::now().to_rfc3339(),
            probe_type,
        }
    }
}

impl Default for HealthRegistry {
    fn default() -> Self {
        Self::new()
    }
}

/// Run a health check with a timeout.
async fn run_check_with_timeout(check: &dyn HealthCheck, timeout: Duration) -> HealthStatus {
    match tokio::time::timeout(timeout, check.check()).await {
        Ok(status) => status,
        Err(_) => HealthStatus::Unhealthy {
            reason: format!("check timed out after {}ms", timeout.as_millis()),
        },
    }
}

/// Always healthy check.
pub struct AlwaysHealthy;

#[async_trait]
impl HealthCheck for AlwaysHealthy {
    fn name(&self) -> &str {
        "always_healthy"
    }

    async fn check(&self) -> HealthStatus {
        HealthStatus::Healthy
    }
}

/// Always degraded check.
pub struct AlwaysDegraded {
    reason: String,
}

impl AlwaysDegraded {
    pub fn new(reason: impl Into<String>) -> Self {
        Self {
            reason: reason.into(),
        }
    }
}

#[async_trait]
impl HealthCheck for AlwaysDegraded {
    fn name(&self) -> &str {
        "always_degraded"
    }

    async fn check(&self) -> HealthStatus {
        HealthStatus::Degraded {
            reason: self.reason.clone(),
        }
    }
}

/// Always unhealthy check.
pub struct AlwaysUnhealthy {
    reason: String,
}

impl AlwaysUnhealthy {
    pub fn new(reason: impl Into<String>) -> Self {
        Self {
            reason: reason.into(),
        }
    }
}

#[async_trait]
impl HealthCheck for AlwaysUnhealthy {
    fn name(&self) -> &str {
        "always_unhealthy"
    }

    async fn check(&self) -> HealthStatus {
        HealthStatus::Unhealthy {
            reason: self.reason.clone(),
        }
    }
}

/// Check that always times out.
pub struct AlwaysTimeout {
    delay: Duration,
}

impl AlwaysTimeout {
    pub fn new(delay: Duration) -> Self {
        Self { delay }
    }
}

#[async_trait]
impl HealthCheck for AlwaysTimeout {
    fn name(&self) -> &str {
        "always_timeout"
    }

    async fn check(&self) -> HealthStatus {
        tokio::time::sleep(self.delay).await;
        HealthStatus::Healthy
    }

    fn timeout(&self) -> Duration {
        Duration::from_millis(10)
    }
}

/// Check that is optional for readiness.
pub struct OptionalCheck;

#[async_trait]
impl HealthCheck for OptionalCheck {
    fn name(&self) -> &str {
        "optional_check"
    }

    async fn check(&self) -> HealthStatus {
        HealthStatus::Healthy
    }

    fn required_for_readiness(&self) -> bool {
        false
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_health_status_is_healthy() {
        assert!(HealthStatus::Healthy.is_healthy());
        assert!(
            !HealthStatus::Degraded {
                reason: "test".to_string()
            }
            .is_healthy()
        );
        assert!(
            !HealthStatus::Unhealthy {
                reason: "test".to_string()
            }
            .is_healthy()
        );
    }

    #[test]
    fn test_health_status_is_degraded() {
        assert!(!HealthStatus::Healthy.is_degraded());
        assert!(
            HealthStatus::Degraded {
                reason: "test".to_string()
            }
            .is_degraded()
        );
        assert!(
            !HealthStatus::Unhealthy {
                reason: "test".to_string()
            }
            .is_degraded()
        );
    }

    #[test]
    fn test_health_status_is_unhealthy() {
        assert!(!HealthStatus::Healthy.is_unhealthy());
        assert!(
            !HealthStatus::Degraded {
                reason: "test".to_string()
            }
            .is_unhealthy()
        );
        assert!(
            HealthStatus::Unhealthy {
                reason: "test".to_string()
            }
            .is_unhealthy()
        );
    }

    #[test]
    fn test_health_status_display() {
        assert_eq!(format!("{}", HealthStatus::Healthy), "healthy");
        assert_eq!(
            format!(
                "{}",
                HealthStatus::Degraded {
                    reason: "slow".to_string()
                }
            ),
            "degraded: slow"
        );
        assert_eq!(
            format!(
                "{}",
                HealthStatus::Unhealthy {
                    reason: "down".to_string()
                }
            ),
            "unhealthy: down"
        );
    }

    #[test]
    fn test_health_status_http_codes() {
        assert_eq!(HealthStatus::Healthy.to_http_status(), 200);
        assert_eq!(
            HealthStatus::Degraded {
                reason: "slow".to_string()
            }
            .to_http_status(),
            200
        );
        assert_eq!(
            HealthStatus::Unhealthy {
                reason: "down".to_string()
            }
            .to_http_status(),
            503
        );
    }

    #[tokio::test]
    async fn test_always_healthy() {
        let check = AlwaysHealthy;
        assert_eq!(check.name(), "always_healthy");
        assert_eq!(check.check().await, HealthStatus::Healthy);
    }

    #[tokio::test]
    async fn test_always_degraded() {
        let check = AlwaysDegraded::new("slow");
        assert_eq!(check.name(), "always_degraded");
        let status = check.check().await;
        assert!(status.is_degraded());
    }

    #[tokio::test]
    async fn test_always_unhealthy() {
        let check = AlwaysUnhealthy::new("down");
        assert_eq!(check.name(), "always_unhealthy");
        let status = check.check().await;
        assert!(status.is_unhealthy());
    }

    #[tokio::test]
    async fn test_always_timeout() {
        let check = AlwaysTimeout::new(Duration::from_millis(100));
        assert_eq!(check.name(), "always_timeout");
        let status = check.check().await;
        assert!(status.is_healthy());
    }

    #[tokio::test]
    async fn native_sync_health_requires_an_available_endpoint() {
        let check = ThingdSyncHealth::new(ThingdSyncMode::Native);
        assert!(check.check().await.is_degraded());
        let check = ThingdSyncHealth::new(ThingdSyncMode::Native).with_native_availability(true);
        assert!(check.check().await.is_healthy());
        assert!(!check.required_for_readiness());
    }

    #[tokio::test]
    async fn test_health_registry_empty() {
        let registry = HealthRegistry::new();
        let report = registry.check_all().await;
        assert_eq!(report.status, HealthStatus::Healthy);
        assert!(report.checks.is_empty());
    }

    #[tokio::test]
    async fn test_health_registry_healthy() {
        let mut registry = HealthRegistry::new();
        registry.register(Arc::new(AlwaysHealthy));
        let report = registry.check_all().await;
        assert_eq!(report.status, HealthStatus::Healthy);
        assert_eq!(report.checks.len(), 1);
        assert_eq!(report.checks[0].status, HealthStatus::Healthy);
    }

    #[tokio::test]
    async fn test_health_registry_degraded() {
        let mut registry = HealthRegistry::new();
        registry.register(Arc::new(AlwaysHealthy));
        registry.register(Arc::new(AlwaysDegraded::new("slow")));
        let report = registry.check_all().await;
        assert!(report.status.is_degraded());
        assert_eq!(report.checks.len(), 2);
    }

    #[tokio::test]
    async fn test_health_registry_unhealthy() {
        let mut registry = HealthRegistry::new();
        registry.register(Arc::new(AlwaysHealthy));
        registry.register(Arc::new(AlwaysUnhealthy::new("down")));
        let report = registry.check_all().await;
        assert!(report.status.is_unhealthy());
        assert_eq!(report.checks.len(), 2);
    }

    #[tokio::test]
    async fn test_health_registry_timeout() {
        let mut registry = HealthRegistry::new();
        registry.register(Arc::new(AlwaysTimeout::new(Duration::from_millis(100))));
        let report = registry.check_all().await;
        assert!(report.status.is_unhealthy());
    }

    #[tokio::test]
    async fn test_readiness_skips_optional() {
        let mut registry = HealthRegistry::new();
        registry.register(Arc::new(AlwaysHealthy));
        registry.register(Arc::new(OptionalCheck));

        let liveness = registry.check_liveness().await;
        assert_eq!(liveness.checks.len(), 2);

        let readiness = registry.check_readiness().await;
        assert_eq!(readiness.checks.len(), 1);
        assert_eq!(readiness.checks[0].name, "always_healthy");
    }

    #[test]
    fn test_check_result_serialization() {
        let result = CheckResult {
            name: "test".to_string(),
            status: HealthStatus::Healthy,
            duration_ms: 10,
        };
        let json = serde_json::to_string(&result).unwrap();
        assert!(json.contains("test"));
        assert!(json.contains("healthy"));
    }

    #[test]
    fn test_health_report_serialization() {
        let report = HealthReport {
            status: HealthStatus::Healthy,
            checks: vec![],
            timestamp: "2024-01-01T00:00:00Z".to_string(),
            probe_type: ProbeType::Liveness,
        };
        let json = serde_json::to_string(&report).unwrap();
        assert!(json.contains("healthy"));
        assert!(json.contains("liveness"));
    }
}