isb_apps/monitor/
mod.rs

1//! Uptime monitors: what users see of an org's apps, checked from outside
2//! the app every interval, with history, incidents and notifications.
3//!
4//! A monitor is an HTTP(S) request (status range, keyword present or
5//! absent, headers from secrets, redirects, a certificate expiry warning),
6//! a TCP connect, or an **app** monitor that follows an app by reference:
7//! its served domain's public URL, or (with no domain, or when the domain
8//! sits behind Cloudflare Access without a service token) the app's own
9//! endpoint. A **service** monitor does the same for one service of a
10//! compose stack. Every app with a served domain gets an app monitor of its
11//! own (`app-<name>`, `auto`), and every compose stack service with one a
12//! service monitor (`stack-<stack>-<service>`, `auto`, see [`auto`]), unless
13//! the org opts out or excludes it.
14//!
15//! Checks run in the daemon that runs the org's apps ([`service`]); state,
16//! thresholds and flap damping are in [`state`], the probes in [`probe`],
17//! history in [`store`]. Channels hear `monitor.down`, `monitor.up` and
18//! `monitor.cert_expiring` through the notifier like every other event.
19//!
20//! Definitions are kept in `<state>/orgs/<org>/monitors/monitors.json`, the
21//! org's settings beside them in `settings.json`.
22
23pub mod auto;
24mod health_path;
25pub mod heartbeat;
26pub mod probe;
27pub mod service;
28pub mod state;
29pub mod store;
30mod target;
31mod view;
32
33use std::path::{Path, PathBuf};
34
35use serde::{Deserialize, Serialize};
36
37use crate::error::{Error, Result};
38use crate::org::OrgId;
39pub use service::Monitors;
40
41/// Monitors per org at most.
42pub const MAX_PER_ORG: usize = 200;
43/// The shortest interval, seconds.
44pub const MIN_INTERVAL: u64 = 30;
45/// The prefix of an app's own monitor.
46pub const AUTO_PREFIX: &str = "app-";
47/// The prefix of a compose stack service's own monitor.
48pub const STACK_PREFIX: &str = "stack-";
49/// The org secrets an app monitor presents to Cloudflare Access, when both
50/// exist and the monitor sets no Access headers of its own.
51pub const ACCESS_ID_SECRET: &str = "CF_ACCESS_CLIENT_ID";
52pub const ACCESS_SECRET_SECRET: &str = "CF_ACCESS_CLIENT_SECRET";
53
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
55#[serde(rename_all = "lowercase")]
56pub enum Kind {
57    Http,
58    Tcp,
59    App,
60    Service,
61}
62
63impl Kind {
64    pub fn as_str(self) -> &'static str {
65        match self {
66            Kind::Http => "http",
67            Kind::Tcp => "tcp",
68            Kind::App => "app",
69            Kind::Service => "service",
70        }
71    }
72}
73
74/// A request header: a plain `value`, or the value of org secret `secret`
75/// (read at check time, never shown).
76#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
77#[serde(deny_unknown_fields)]
78pub struct Header {
79    pub name: String,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub value: Option<String>,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub secret: Option<String>,
84}
85
86/// One monitor. Fields that do not apply to its type are refused.
87#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
88#[serde(deny_unknown_fields)]
89pub struct Monitor {
90    pub name: String,
91    #[serde(rename = "type")]
92    pub kind: Kind,
93    /// http: the URL.
94    #[serde(default, skip_serializing_if = "Option::is_none")]
95    pub url: Option<String>,
96    /// tcp: host and port.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub host: Option<String>,
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub port: Option<u16>,
101    /// app: the app, by name.
102    #[serde(default, skip_serializing_if = "Option::is_none")]
103    pub app: Option<String>,
104    /// service: the compose stack, by name.
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub stack: Option<String>,
107    /// service: the stack's service, by name.
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub service: Option<String>,
110    /// app, service: which of its domains (default: the first one served).
111    #[serde(default, skip_serializing_if = "Option::is_none")]
112    pub domain: Option<String>,
113    /// app, service: the path to request (default: the path the service's healthcheck requests over HTTP, else the domain's path).
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub path: Option<String>,
116    /// `GET` or `HEAD`.
117    #[serde(default = "get")]
118    pub method: String,
119    /// Status codes that count as up: `200-399`, `200,204`, `200-299,301`.
120    #[serde(default = "expected")]
121    pub expected_status: String,
122    /// The body must contain this.
123    #[serde(default, skip_serializing_if = "Option::is_none")]
124    pub keyword: Option<String>,
125    /// The body must not contain this.
126    #[serde(default, skip_serializing_if = "Option::is_none")]
127    pub keyword_absent: Option<String>,
128    #[serde(default)]
129    pub follow_redirects: bool,
130    #[serde(default, skip_serializing_if = "Vec::is_empty")]
131    pub headers: Vec<Header>,
132    /// Seconds between checks (at least 30).
133    #[serde(default = "interval")]
134    pub interval: u64,
135    /// Seconds one check may take.
136    #[serde(default = "timeout")]
137    pub timeout: u64,
138    /// Failed checks in a row that make it down.
139    #[serde(default = "two")]
140    pub failure_threshold: u32,
141    /// Successful checks in a row that make it up again.
142    #[serde(default = "two")]
143    pub recovery_threshold: u32,
144    /// Warn this many days before an HTTPS certificate expires (0: never).
145    #[serde(default = "cert_days")]
146    pub cert_expiry_days: u32,
147    #[serde(default)]
148    pub paused: bool,
149    /// Made for an app (`app-<name>`) or a compose stack service
150    /// (`stack-<stack>-<service>`) with a served domain.
151    #[serde(default)]
152    pub auto: bool,
153    #[serde(default)]
154    pub created_at: u64,
155    #[serde(default)]
156    pub updated_at: u64,
157}
158
159fn get() -> String {
160    "GET".into()
161}
162fn expected() -> String {
163    "200-399".into()
164}
165fn interval() -> u64 {
166    60
167}
168fn timeout() -> u64 {
169    10
170}
171fn two() -> u32 {
172    2
173}
174fn cert_days() -> u32 {
175    14
176}
177
178/// An org's monitoring settings.
179#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
180#[serde(deny_unknown_fields)]
181pub struct Settings {
182    /// Give every app with a served domain a monitor of its own.
183    #[serde(default = "yes")]
184    pub auto_monitors: bool,
185    /// Apps that get no monitor of their own.
186    #[serde(default, skip_serializing_if = "Vec::is_empty")]
187    pub exclude_apps: Vec<String>,
188    /// Compose stack services that get no monitor of their own, as
189    /// `<stack>/<service>`.
190    #[serde(default, skip_serializing_if = "Vec::is_empty")]
191    pub exclude_services: Vec<String>,
192}
193
194fn yes() -> bool {
195    true
196}
197
198impl Default for Settings {
199    fn default() -> Settings {
200        Settings {
201            auto_monitors: true,
202            exclude_apps: Vec::new(),
203            exclude_services: Vec::new(),
204        }
205    }
206}
207
208/// A monitor name: [a-z0-9-], a letter first, at most 63.
209pub fn validate_name(n: &str) -> Result<()> {
210    let ok = !n.is_empty()
211        && n.len() <= 63
212        && n.starts_with(|c: char| c.is_ascii_lowercase())
213        && n.chars()
214            .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-');
215    if ok {
216        Ok(())
217    } else {
218        Err(Error::invalid(format!(
219            "monitor name {n:?}: [a-z0-9-], starting with a letter, at most 63 characters"
220        )))
221    }
222}
223
224/// A compose service's name, as a service monitor refers to it.
225pub fn validate_service_name(s: &str) -> Result<()> {
226    let ok = !s.is_empty()
227        && s.len() <= 63
228        && s.chars()
229            .all(|c| c.is_ascii_alphanumeric() || "._-".contains(c));
230    if ok {
231        Ok(())
232    } else {
233        Err(Error::invalid(format!(
234            "service {s:?}: a compose service's name ([A-Za-z0-9._-], at most 63)"
235        )))
236    }
237}
238
239/// Parse `200-399`, `200,204,300-399` into inclusive ranges.
240pub fn parse_status(s: &str) -> Result<Vec<(u16, u16)>> {
241    let bad = || {
242        Error::invalid(format!(
243            "expected_status {s:?}: codes and ranges like 200-399 or 200,204"
244        ))
245    };
246    let mut out = Vec::new();
247    for part in s.split(',').map(str::trim) {
248        let (a, b) = part.split_once('-').unwrap_or((part, part));
249        let a: u16 = a.trim().parse().map_err(|_| bad())?;
250        let b: u16 = b.trim().parse().map_err(|_| bad())?;
251        if !(100..=599).contains(&a) || !(100..=599).contains(&b) || a > b {
252            return Err(bad());
253        }
254        out.push((a, b));
255    }
256    if out.is_empty() || out.len() > 10 {
257        return Err(bad());
258    }
259    Ok(out)
260}
261
262/// Headers isb sets itself, or that would change how the request is framed.
263const RESERVED_HEADERS: &[&str] = &[
264    "host",
265    "content-length",
266    "transfer-encoding",
267    "connection",
268    "upgrade",
269];
270
271fn validate_headers(hs: &[Header]) -> Result<()> {
272    if hs.len() > 10 {
273        return Err(Error::invalid("at most 10 headers"));
274    }
275    for h in hs {
276        let n = &h.name;
277        let token = !n.is_empty()
278            && n.len() <= 64
279            && n.chars()
280                .all(|c| c.is_ascii_alphanumeric() || "!#$%&'*+-.^_`|~".contains(c));
281        if !token || RESERVED_HEADERS.contains(&n.to_ascii_lowercase().as_str()) {
282            return Err(Error::invalid(format!("header name {n:?}")));
283        }
284        match (&h.value, &h.secret) {
285            (Some(v), None) if v.len() <= 1024 && !v.contains(['\r', '\n']) => {}
286            (None, Some(s)) => crate::secrets::validate_name(s)?,
287            _ => {
288                return Err(Error::invalid(format!(
289                    "header {n}: a value (without line breaks, at most 1024) or a secret, not both"
290                )));
291            }
292        }
293    }
294    Ok(())
295}
296
297fn validate_tcp_host(h: &str) -> Result<()> {
298    let ok = !h.is_empty()
299        && h.len() <= 253
300        && h.chars()
301            .all(|c| c.is_ascii_alphanumeric() || ".-:[]".contains(c));
302    if ok {
303        Ok(())
304    } else {
305        Err(Error::invalid(format!(
306            "host {h:?} is not a host name or address"
307        )))
308    }
309}
310
311impl Monitor {
312    /// A bare monitor of `kind` with every default.
313    pub fn new(name: &str, kind: Kind) -> Monitor {
314        serde_json::from_value(serde_json::json!({"name": name, "type": kind}))
315            .expect("defaults deserialize")
316    }
317
318    /// Check everything but whether an app or secret exists.
319    pub fn validate(&self) -> Result<()> {
320        validate_name(&self.name)?;
321        self.validate_target()?;
322        if !(MIN_INTERVAL..=86_400).contains(&self.interval) {
323            return Err(Error::invalid(format!(
324                "interval: {MIN_INTERVAL} to 86400 seconds"
325            )));
326        }
327        if !(1..=60).contains(&self.timeout) || self.timeout >= self.interval {
328            return Err(Error::invalid(
329                "timeout: 1 to 60 seconds, under the interval",
330            ));
331        }
332        for (what, n) in [
333            ("failure_threshold", self.failure_threshold),
334            ("recovery_threshold", self.recovery_threshold),
335        ] {
336            if !(1..=10).contains(&n) {
337                return Err(Error::invalid(format!("{what}: 1 to 10 checks")));
338            }
339        }
340        if self.cert_expiry_days > 365 {
341            return Err(Error::invalid("cert_expiry_days: 0 to 365"));
342        }
343        Ok(())
344    }
345
346    fn validate_target(&self) -> Result<()> {
347        let http_only = self.keyword.is_some()
348            || self.keyword_absent.is_some()
349            || !self.headers.is_empty()
350            || self.follow_redirects;
351        match self.kind {
352            Kind::Http => {
353                let url = self
354                    .url
355                    .as_deref()
356                    .ok_or_else(|| Error::invalid("an http monitor needs url"))?;
357                crate::net::parse_url(url).map_err(|e| Error::invalid(format!("url: {e}")))?;
358                self.refuse(&[
359                    ("host", self.host.is_some()),
360                    ("port", self.port.is_some()),
361                    ("app", self.app.is_some()),
362                    (
363                        "stack or service",
364                        self.stack.is_some() || self.service.is_some(),
365                    ),
366                    ("domain", self.domain.is_some()),
367                    ("path", self.path.is_some()),
368                ])?;
369            }
370            Kind::Tcp => {
371                validate_tcp_host(self.host.as_deref().unwrap_or_default())?;
372                if self.port.is_none_or(|p| p == 0) {
373                    return Err(Error::invalid("a tcp monitor needs host and port"));
374                }
375                self.refuse(&[
376                    ("url", self.url.is_some()),
377                    ("app", self.app.is_some()),
378                    (
379                        "stack or service",
380                        self.stack.is_some() || self.service.is_some(),
381                    ),
382                    ("domain", self.domain.is_some()),
383                    ("path", self.path.is_some()),
384                    ("keyword, headers or follow_redirects", http_only),
385                ])?;
386            }
387            Kind::App => {
388                let app = self.app.as_deref().unwrap_or_default();
389                crate::app::validate_app_name(app)
390                    .map_err(|_| Error::invalid(format!("app {app:?}: an app's name")))?;
391                self.validate_path()?;
392                self.refuse(&[
393                    ("url", self.url.is_some()),
394                    ("host", self.host.is_some()),
395                    ("port", self.port.is_some()),
396                    (
397                        "stack or service",
398                        self.stack.is_some() || self.service.is_some(),
399                    ),
400                ])?;
401            }
402            Kind::Service => {
403                let stack = self.stack.as_deref().unwrap_or_default();
404                crate::stack::validate_stack_name(stack)
405                    .map_err(|_| Error::invalid(format!("stack {stack:?}: a stack's name")))?;
406                validate_service_name(self.service.as_deref().unwrap_or_default())?;
407                self.validate_path()?;
408                self.refuse(&[
409                    ("url", self.url.is_some()),
410                    ("host", self.host.is_some()),
411                    ("port", self.port.is_some()),
412                    ("app", self.app.is_some()),
413                ])?;
414            }
415        }
416        if self.kind != Kind::Tcp {
417            if !matches!(self.method.as_str(), "GET" | "HEAD") {
418                return Err(Error::invalid("method: GET or HEAD"));
419            }
420            parse_status(&self.expected_status)?;
421            for k in [&self.keyword, &self.keyword_absent].into_iter().flatten() {
422                if k.is_empty() || k.len() > 200 {
423                    return Err(Error::invalid("keywords: 1 to 200 characters"));
424                }
425            }
426            validate_headers(&self.headers)?;
427        }
428        Ok(())
429    }
430
431    fn validate_path(&self) -> Result<()> {
432        match &self.path {
433            Some(p) if !p.starts_with('/') || p.len() > 1024 || p.contains(char::is_whitespace) => {
434                Err(Error::invalid("path: starts with /, no spaces"))
435            }
436            _ => Ok(()),
437        }
438    }
439
440    /// Does it follow something by reference (an app or a stack service)?
441    pub fn follows(&self) -> bool {
442        matches!(self.kind, Kind::App | Kind::Service)
443    }
444
445    fn refuse(&self, fields: &[(&str, bool)]) -> Result<()> {
446        match fields.iter().find(|(_, set)| *set) {
447            Some((f, _)) => Err(Error::invalid(format!(
448                "{f} does not apply to a {} monitor",
449                self.kind.as_str()
450            ))),
451            None => Ok(()),
452        }
453    }
454
455    /// The secrets the monitor reads.
456    pub fn secrets(&self) -> Vec<&str> {
457        self.headers
458            .iter()
459            .filter_map(|h| h.secret.as_deref())
460            .collect()
461    }
462
463    /// What it checks, in a few words.
464    pub fn target(&self) -> String {
465        match self.kind {
466            Kind::Http => probe::display_url(self.url.as_deref().unwrap_or_default()),
467            Kind::Tcp => format!(
468                "{}:{}",
469                self.host.as_deref().unwrap_or_default(),
470                self.port.unwrap_or_default()
471            ),
472            Kind::App => format!("app {}", self.app.as_deref().unwrap_or_default()),
473            Kind::Service => format!(
474                "service {}/{}",
475                self.stack.as_deref().unwrap_or_default(),
476                self.service.as_deref().unwrap_or_default()
477            ),
478        }
479    }
480}
481
482/// Where an org's monitors live.
483pub fn dir(state: &Path, org: &OrgId) -> PathBuf {
484    org.dir(state).join("monitors")
485}
486
487pub(crate) fn read_json<T: for<'de> Deserialize<'de> + Default>(p: &Path) -> Result<T> {
488    match std::fs::read(p) {
489        Ok(b) => {
490            serde_json::from_slice(&b).map_err(|e| Error::invalid(format!("{}: {e}", p.display())))
491        }
492        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(T::default()),
493        Err(e) => Err(e.into()),
494    }
495}
496
497pub(crate) fn write_json<T: Serialize>(p: &Path, v: &T) -> Result<()> {
498    if let Some(d) = p.parent() {
499        std::fs::create_dir_all(d)?;
500    }
501    let tmp = p.with_extension("tmp");
502    std::fs::write(&tmp, serde_json::to_vec_pretty(v)?)?;
503    std::fs::rename(&tmp, p)?;
504    Ok(())
505}
506
507/// A duration in words: `4m 12s`, `2h 5m`, `3d 1h`.
508pub fn human(ms: u64) -> String {
509    let s = ms / 1000;
510    match s {
511        0..60 => format!("{s}s"),
512        60..3600 => format!("{}m {}s", s / 60, s % 60),
513        3600..86_400 => format!("{}h {}m", s / 3600, s % 3600 / 60),
514        _ => format!("{}d {}h", s / 86_400, s % 86_400 / 3600),
515    }
516}
517
518#[cfg(test)]
519mod tests {
520    use super::*;
521    use serde_json::json;
522
523    fn m(v: serde_json::Value) -> Result<Monitor> {
524        let m: Monitor = serde_json::from_value(v).map_err(|e| Error::invalid(e.to_string()))?;
525        m.validate()?;
526        Ok(m)
527    }
528
529    #[test]
530    fn defaults_and_validation() {
531        let h =
532            m(json!({"name": "shop", "type": "http", "url": "https://shop.example.com/"})).unwrap();
533        assert_eq!(
534            (
535                h.interval,
536                h.timeout,
537                h.failure_threshold,
538                h.recovery_threshold
539            ),
540            (60, 10, 2, 2)
541        );
542        assert_eq!(
543            (
544                h.method.as_str(),
545                h.expected_status.as_str(),
546                h.cert_expiry_days
547            ),
548            ("GET", "200-399", 14)
549        );
550        assert!(m(json!({"name": "x", "type": "http"})).is_err());
551        assert!(m(json!({"name": "x", "type": "http", "url": "ftp://a"})).is_err());
552        assert!(
553            m(json!({"name": "x", "type": "http", "url": "https://a/", "interval": 10})).is_err()
554        );
555        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "interval": 30, "timeout": 30})).is_err());
556        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "port": 80})).is_err());
557        assert!(
558            m(json!({"name": "x", "type": "http", "url": "https://a/", "method": "POST"})).is_err()
559        );
560        assert!(
561            m(json!({"name": "x", "type": "http", "url": "https://a/", "failure_threshold": 0}))
562                .is_err()
563        );
564        assert!(m(json!({"name": "X", "type": "http", "url": "https://a/"})).is_err());
565        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "bogus": 1})).is_err());
566        m(json!({"name": "db", "type": "tcp", "host": "db.example.com", "port": 5432})).unwrap();
567        assert!(m(json!({"name": "db", "type": "tcp", "host": "db.example.com"})).is_err());
568        assert!(m(json!({"name": "db", "type": "tcp", "host": "a b", "port": 1})).is_err());
569        assert!(
570            m(json!({"name": "db", "type": "tcp", "host": "a", "port": 1, "keyword": "x"}))
571                .is_err()
572        );
573        m(json!({"name": "web", "type": "app", "app": "web", "path": "/healthz"})).unwrap();
574        assert!(m(json!({"name": "web", "type": "app", "app": "web", "path": "healthz"})).is_err());
575        assert!(
576            m(json!({"name": "web", "type": "app", "app": "web", "url": "https://a/"})).is_err()
577        );
578    }
579
580    #[test]
581    fn headers_and_status_ranges() {
582        let ok = json!({"name": "a", "type": "http", "url": "https://a/", "headers": [
583            {"name": "CF-Access-Client-Id", "secret": "CF_ID"}, {"name": "Accept", "value": "text/html"}]});
584        assert_eq!(m(ok).unwrap().secrets(), vec!["CF_ID"]);
585        for bad in [
586            json!([{"name": "Host", "value": "x"}]),
587            json!([{"name": "X", "value": "a\r\nb: c"}]),
588            json!([{"name": "X", "value": "a", "secret": "S"}]),
589            json!([{"name": "X"}]),
590            json!([{"name": "Bad Name", "value": "a"}]),
591        ] {
592            assert!(
593                m(json!({"name": "a", "type": "http", "url": "https://a/", "headers": bad}))
594                    .is_err(),
595                "{bad}"
596            );
597        }
598        assert_eq!(parse_status("200-399").unwrap(), vec![(200, 399)]);
599        assert_eq!(
600            parse_status("200, 204,300-301").unwrap(),
601            vec![(200, 200), (204, 204), (300, 301)]
602        );
603        for bad in ["", "abc", "399-200", "99", "200-700", "200,"] {
604            assert!(parse_status(bad).is_err(), "{bad}");
605        }
606    }
607
608    #[test]
609    fn words() {
610        assert_eq!(human(4_000), "4s");
611        assert_eq!(human(252_000), "4m 12s");
612        assert_eq!(human(7_500_000), "2h 5m");
613        assert_eq!(human(266_400_000), "3d 2h");
614        assert_eq!(Monitor::new("x", Kind::Tcp).interval, 60);
615    }
616}
isb_apps/monitor/mod.rs

isb_apps/monitor/
mod.rs
