isb_apps/monitor/
mod.rs

1//! Uptime monitors: what users see of an org's apps, checked from outside
2//! the app every interval, with history, incidents and notifications.
3//!
4//! A monitor is an HTTP(S) request (status range, keyword present or
5//! absent, headers from secrets, redirects, a certificate expiry warning),
6//! a TCP connect, or an **app** monitor that follows an app by reference:
7//! its served domain's public URL, or (with no domain, or when the domain
8//! sits behind Cloudflare Access without a service token) the app's own
9//! endpoint. Every app with a served domain gets an app monitor of its own
10//! (`app-<name>`, `auto`), unless the org or the app opts out.
11//!
12//! Checks run in the daemon that runs the org's apps ([`service`]); state,
13//! thresholds and flap damping are in [`state`], the probes in [`probe`],
14//! history in [`store`]. Channels hear `monitor.down`, `monitor.up` and
15//! `monitor.cert_expiring` through the notifier like every other event.
16//!
17//! Definitions are kept in `<state>/orgs/<org>/monitors/monitors.json`, the
18//! org's settings beside them in `settings.json`.
19
20pub mod heartbeat;
21pub mod probe;
22pub mod service;
23pub mod state;
24pub mod store;
25mod target;
26mod view;
27
28use std::path::{Path, PathBuf};
29
30use serde::{Deserialize, Serialize};
31
32use crate::error::{Error, Result};
33use crate::org::OrgId;
34pub use service::Monitors;
35
36/// Monitors per org at most.
37pub const MAX_PER_ORG: usize = 200;
38/// The shortest interval, seconds.
39pub const MIN_INTERVAL: u64 = 30;
40/// The prefix of an app's own monitor.
41pub const AUTO_PREFIX: &str = "app-";
42/// The org secrets an app monitor presents to Cloudflare Access, when both
43/// exist and the monitor sets no Access headers of its own.
44pub const ACCESS_ID_SECRET: &str = "CF_ACCESS_CLIENT_ID";
45pub const ACCESS_SECRET_SECRET: &str = "CF_ACCESS_CLIENT_SECRET";
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
48#[serde(rename_all = "lowercase")]
49pub enum Kind {
50    Http,
51    Tcp,
52    App,
53}
54
55impl Kind {
56    pub fn as_str(self) -> &'static str {
57        match self {
58            Kind::Http => "http",
59            Kind::Tcp => "tcp",
60            Kind::App => "app",
61        }
62    }
63}
64
65/// A request header: a plain `value`, or the value of org secret `secret`
66/// (read at check time, never shown).
67#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
68#[serde(deny_unknown_fields)]
69pub struct Header {
70    pub name: String,
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub value: Option<String>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub secret: Option<String>,
75}
76
77/// One monitor. Fields that do not apply to its type are refused.
78#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
79#[serde(deny_unknown_fields)]
80pub struct Monitor {
81    pub name: String,
82    #[serde(rename = "type")]
83    pub kind: Kind,
84    /// http: the URL.
85    #[serde(default, skip_serializing_if = "Option::is_none")]
86    pub url: Option<String>,
87    /// tcp: host and port.
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub host: Option<String>,
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub port: Option<u16>,
92    /// app: the app, by name.
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub app: Option<String>,
95    /// app: which of its domains (default: the first one served).
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub domain: Option<String>,
98    /// app: the path to request (default: the domain's path).
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub path: Option<String>,
101    /// `GET` or `HEAD`.
102    #[serde(default = "get")]
103    pub method: String,
104    /// Status codes that count as up: `200-399`, `200,204`, `200-299,301`.
105    #[serde(default = "expected")]
106    pub expected_status: String,
107    /// The body must contain this.
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub keyword: Option<String>,
110    /// The body must not contain this.
111    #[serde(default, skip_serializing_if = "Option::is_none")]
112    pub keyword_absent: Option<String>,
113    #[serde(default)]
114    pub follow_redirects: bool,
115    #[serde(default, skip_serializing_if = "Vec::is_empty")]
116    pub headers: Vec<Header>,
117    /// Seconds between checks (at least 30).
118    #[serde(default = "interval")]
119    pub interval: u64,
120    /// Seconds one check may take.
121    #[serde(default = "timeout")]
122    pub timeout: u64,
123    /// Failed checks in a row that make it down.
124    #[serde(default = "two")]
125    pub failure_threshold: u32,
126    /// Successful checks in a row that make it up again.
127    #[serde(default = "two")]
128    pub recovery_threshold: u32,
129    /// Warn this many days before an HTTPS certificate expires (0: never).
130    #[serde(default = "cert_days")]
131    pub cert_expiry_days: u32,
132    #[serde(default)]
133    pub paused: bool,
134    /// Made for an app with a served domain (`app-<name>`).
135    #[serde(default)]
136    pub auto: bool,
137    #[serde(default)]
138    pub created_at: u64,
139    #[serde(default)]
140    pub updated_at: u64,
141}
142
143fn get() -> String {
144    "GET".into()
145}
146fn expected() -> String {
147    "200-399".into()
148}
149fn interval() -> u64 {
150    60
151}
152fn timeout() -> u64 {
153    10
154}
155fn two() -> u32 {
156    2
157}
158fn cert_days() -> u32 {
159    14
160}
161
162/// An org's monitoring settings.
163#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
164#[serde(deny_unknown_fields)]
165pub struct Settings {
166    /// Give every app with a served domain a monitor of its own.
167    #[serde(default = "yes")]
168    pub auto_monitors: bool,
169    /// Apps that get no monitor of their own.
170    #[serde(default, skip_serializing_if = "Vec::is_empty")]
171    pub exclude_apps: Vec<String>,
172}
173
174fn yes() -> bool {
175    true
176}
177
178impl Default for Settings {
179    fn default() -> Settings {
180        Settings {
181            auto_monitors: true,
182            exclude_apps: Vec::new(),
183        }
184    }
185}
186
187/// A monitor name: [a-z0-9-], a letter first, at most 63.
188pub fn validate_name(n: &str) -> Result<()> {
189    let ok = !n.is_empty()
190        && n.len() <= 63
191        && n.starts_with(|c: char| c.is_ascii_lowercase())
192        && n.chars()
193            .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-');
194    if ok {
195        Ok(())
196    } else {
197        Err(Error::invalid(format!(
198            "monitor name {n:?}: [a-z0-9-], starting with a letter, at most 63 characters"
199        )))
200    }
201}
202
203/// Parse `200-399`, `200,204,300-399` into inclusive ranges.
204pub fn parse_status(s: &str) -> Result<Vec<(u16, u16)>> {
205    let bad = || {
206        Error::invalid(format!(
207            "expected_status {s:?}: codes and ranges like 200-399 or 200,204"
208        ))
209    };
210    let mut out = Vec::new();
211    for part in s.split(',').map(str::trim) {
212        let (a, b) = part.split_once('-').unwrap_or((part, part));
213        let a: u16 = a.trim().parse().map_err(|_| bad())?;
214        let b: u16 = b.trim().parse().map_err(|_| bad())?;
215        if !(100..=599).contains(&a) || !(100..=599).contains(&b) || a > b {
216            return Err(bad());
217        }
218        out.push((a, b));
219    }
220    if out.is_empty() || out.len() > 10 {
221        return Err(bad());
222    }
223    Ok(out)
224}
225
226/// Headers isb sets itself, or that would change how the request is framed.
227const RESERVED_HEADERS: &[&str] = &[
228    "host",
229    "content-length",
230    "transfer-encoding",
231    "connection",
232    "upgrade",
233];
234
235fn validate_headers(hs: &[Header]) -> Result<()> {
236    if hs.len() > 10 {
237        return Err(Error::invalid("at most 10 headers"));
238    }
239    for h in hs {
240        let n = &h.name;
241        let token = !n.is_empty()
242            && n.len() <= 64
243            && n.chars()
244                .all(|c| c.is_ascii_alphanumeric() || "!#$%&'*+-.^_`|~".contains(c));
245        if !token || RESERVED_HEADERS.contains(&n.to_ascii_lowercase().as_str()) {
246            return Err(Error::invalid(format!("header name {n:?}")));
247        }
248        match (&h.value, &h.secret) {
249            (Some(v), None) if v.len() <= 1024 && !v.contains(['\r', '\n']) => {}
250            (None, Some(s)) => crate::secrets::validate_name(s)?,
251            _ => {
252                return Err(Error::invalid(format!(
253                    "header {n}: a value (without line breaks, at most 1024) or a secret, not both"
254                )));
255            }
256        }
257    }
258    Ok(())
259}
260
261fn validate_tcp_host(h: &str) -> Result<()> {
262    let ok = !h.is_empty()
263        && h.len() <= 253
264        && h.chars()
265            .all(|c| c.is_ascii_alphanumeric() || ".-:[]".contains(c));
266    if ok {
267        Ok(())
268    } else {
269        Err(Error::invalid(format!(
270            "host {h:?} is not a host name or address"
271        )))
272    }
273}
274
275impl Monitor {
276    /// A bare monitor of `kind` with every default.
277    pub fn new(name: &str, kind: Kind) -> Monitor {
278        serde_json::from_value(serde_json::json!({"name": name, "type": kind}))
279            .expect("defaults deserialize")
280    }
281
282    /// Check everything but whether an app or secret exists.
283    pub fn validate(&self) -> Result<()> {
284        validate_name(&self.name)?;
285        self.validate_target()?;
286        if !(MIN_INTERVAL..=86_400).contains(&self.interval) {
287            return Err(Error::invalid(format!(
288                "interval: {MIN_INTERVAL} to 86400 seconds"
289            )));
290        }
291        if !(1..=60).contains(&self.timeout) || self.timeout >= self.interval {
292            return Err(Error::invalid(
293                "timeout: 1 to 60 seconds, under the interval",
294            ));
295        }
296        for (what, n) in [
297            ("failure_threshold", self.failure_threshold),
298            ("recovery_threshold", self.recovery_threshold),
299        ] {
300            if !(1..=10).contains(&n) {
301                return Err(Error::invalid(format!("{what}: 1 to 10 checks")));
302            }
303        }
304        if self.cert_expiry_days > 365 {
305            return Err(Error::invalid("cert_expiry_days: 0 to 365"));
306        }
307        Ok(())
308    }
309
310    fn validate_target(&self) -> Result<()> {
311        let http_only = self.keyword.is_some()
312            || self.keyword_absent.is_some()
313            || !self.headers.is_empty()
314            || self.follow_redirects;
315        match self.kind {
316            Kind::Http => {
317                let url = self
318                    .url
319                    .as_deref()
320                    .ok_or_else(|| Error::invalid("an http monitor needs url"))?;
321                crate::net::parse_url(url).map_err(|e| Error::invalid(format!("url: {e}")))?;
322                self.refuse(&[
323                    ("host", self.host.is_some()),
324                    ("port", self.port.is_some()),
325                    ("app", self.app.is_some()),
326                    ("domain", self.domain.is_some()),
327                    ("path", self.path.is_some()),
328                ])?;
329            }
330            Kind::Tcp => {
331                validate_tcp_host(self.host.as_deref().unwrap_or_default())?;
332                if self.port.is_none_or(|p| p == 0) {
333                    return Err(Error::invalid("a tcp monitor needs host and port"));
334                }
335                self.refuse(&[
336                    ("url", self.url.is_some()),
337                    ("app", self.app.is_some()),
338                    ("domain", self.domain.is_some()),
339                    ("path", self.path.is_some()),
340                    ("keyword, headers or follow_redirects", http_only),
341                ])?;
342            }
343            Kind::App => {
344                let app = self.app.as_deref().unwrap_or_default();
345                crate::app::validate_app_name(app)
346                    .map_err(|_| Error::invalid(format!("app {app:?}: an app's name")))?;
347                if let Some(p) = &self.path {
348                    if !p.starts_with('/') || p.len() > 1024 || p.contains(char::is_whitespace) {
349                        return Err(Error::invalid("path: starts with /, no spaces"));
350                    }
351                }
352                self.refuse(&[
353                    ("url", self.url.is_some()),
354                    ("host", self.host.is_some()),
355                    ("port", self.port.is_some()),
356                ])?;
357            }
358        }
359        if self.kind != Kind::Tcp {
360            if !matches!(self.method.as_str(), "GET" | "HEAD") {
361                return Err(Error::invalid("method: GET or HEAD"));
362            }
363            parse_status(&self.expected_status)?;
364            for k in [&self.keyword, &self.keyword_absent].into_iter().flatten() {
365                if k.is_empty() || k.len() > 200 {
366                    return Err(Error::invalid("keywords: 1 to 200 characters"));
367                }
368            }
369            validate_headers(&self.headers)?;
370        }
371        Ok(())
372    }
373
374    fn refuse(&self, fields: &[(&str, bool)]) -> Result<()> {
375        match fields.iter().find(|(_, set)| *set) {
376            Some((f, _)) => Err(Error::invalid(format!(
377                "{f} does not apply to a {} monitor",
378                self.kind.as_str()
379            ))),
380            None => Ok(()),
381        }
382    }
383
384    /// The secrets the monitor reads.
385    pub fn secrets(&self) -> Vec<&str> {
386        self.headers
387            .iter()
388            .filter_map(|h| h.secret.as_deref())
389            .collect()
390    }
391
392    /// What it checks, in a few words.
393    pub fn target(&self) -> String {
394        match self.kind {
395            Kind::Http => probe::display_url(self.url.as_deref().unwrap_or_default()),
396            Kind::Tcp => format!(
397                "{}:{}",
398                self.host.as_deref().unwrap_or_default(),
399                self.port.unwrap_or_default()
400            ),
401            Kind::App => format!("app {}", self.app.as_deref().unwrap_or_default()),
402        }
403    }
404}
405
406/// Where an org's monitors live.
407pub fn dir(state: &Path, org: &OrgId) -> PathBuf {
408    org.dir(state).join("monitors")
409}
410
411pub(crate) fn read_json<T: for<'de> Deserialize<'de> + Default>(p: &Path) -> Result<T> {
412    match std::fs::read(p) {
413        Ok(b) => {
414            serde_json::from_slice(&b).map_err(|e| Error::invalid(format!("{}: {e}", p.display())))
415        }
416        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(T::default()),
417        Err(e) => Err(e.into()),
418    }
419}
420
421pub(crate) fn write_json<T: Serialize>(p: &Path, v: &T) -> Result<()> {
422    if let Some(d) = p.parent() {
423        std::fs::create_dir_all(d)?;
424    }
425    let tmp = p.with_extension("tmp");
426    std::fs::write(&tmp, serde_json::to_vec_pretty(v)?)?;
427    std::fs::rename(&tmp, p)?;
428    Ok(())
429}
430
431/// A duration in words: `4m 12s`, `2h 5m`, `3d 1h`.
432pub fn human(ms: u64) -> String {
433    let s = ms / 1000;
434    match s {
435        0..60 => format!("{s}s"),
436        60..3600 => format!("{}m {}s", s / 60, s % 60),
437        3600..86_400 => format!("{}h {}m", s / 3600, s % 3600 / 60),
438        _ => format!("{}d {}h", s / 86_400, s % 86_400 / 3600),
439    }
440}
441
442#[cfg(test)]
443mod tests {
444    use super::*;
445    use serde_json::json;
446
447    fn m(v: serde_json::Value) -> Result<Monitor> {
448        let m: Monitor = serde_json::from_value(v).map_err(|e| Error::invalid(e.to_string()))?;
449        m.validate()?;
450        Ok(m)
451    }
452
453    #[test]
454    fn defaults_and_validation() {
455        let h =
456            m(json!({"name": "shop", "type": "http", "url": "https://shop.example.com/"})).unwrap();
457        assert_eq!(
458            (
459                h.interval,
460                h.timeout,
461                h.failure_threshold,
462                h.recovery_threshold
463            ),
464            (60, 10, 2, 2)
465        );
466        assert_eq!(
467            (
468                h.method.as_str(),
469                h.expected_status.as_str(),
470                h.cert_expiry_days
471            ),
472            ("GET", "200-399", 14)
473        );
474        assert!(m(json!({"name": "x", "type": "http"})).is_err());
475        assert!(m(json!({"name": "x", "type": "http", "url": "ftp://a"})).is_err());
476        assert!(
477            m(json!({"name": "x", "type": "http", "url": "https://a/", "interval": 10})).is_err()
478        );
479        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "interval": 30, "timeout": 30})).is_err());
480        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "port": 80})).is_err());
481        assert!(
482            m(json!({"name": "x", "type": "http", "url": "https://a/", "method": "POST"})).is_err()
483        );
484        assert!(
485            m(json!({"name": "x", "type": "http", "url": "https://a/", "failure_threshold": 0}))
486                .is_err()
487        );
488        assert!(m(json!({"name": "X", "type": "http", "url": "https://a/"})).is_err());
489        assert!(m(json!({"name": "x", "type": "http", "url": "https://a/", "bogus": 1})).is_err());
490        m(json!({"name": "db", "type": "tcp", "host": "db.example.com", "port": 5432})).unwrap();
491        assert!(m(json!({"name": "db", "type": "tcp", "host": "db.example.com"})).is_err());
492        assert!(m(json!({"name": "db", "type": "tcp", "host": "a b", "port": 1})).is_err());
493        assert!(
494            m(json!({"name": "db", "type": "tcp", "host": "a", "port": 1, "keyword": "x"}))
495                .is_err()
496        );
497        m(json!({"name": "web", "type": "app", "app": "web", "path": "/healthz"})).unwrap();
498        assert!(m(json!({"name": "web", "type": "app", "app": "web", "path": "healthz"})).is_err());
499        assert!(
500            m(json!({"name": "web", "type": "app", "app": "web", "url": "https://a/"})).is_err()
501        );
502    }
503
504    #[test]
505    fn headers_and_status_ranges() {
506        let ok = json!({"name": "a", "type": "http", "url": "https://a/", "headers": [
507            {"name": "CF-Access-Client-Id", "secret": "CF_ID"}, {"name": "Accept", "value": "text/html"}]});
508        assert_eq!(m(ok).unwrap().secrets(), vec!["CF_ID"]);
509        for bad in [
510            json!([{"name": "Host", "value": "x"}]),
511            json!([{"name": "X", "value": "a\r\nb: c"}]),
512            json!([{"name": "X", "value": "a", "secret": "S"}]),
513            json!([{"name": "X"}]),
514            json!([{"name": "Bad Name", "value": "a"}]),
515        ] {
516            assert!(
517                m(json!({"name": "a", "type": "http", "url": "https://a/", "headers": bad}))
518                    .is_err(),
519                "{bad}"
520            );
521        }
522        assert_eq!(parse_status("200-399").unwrap(), vec![(200, 399)]);
523        assert_eq!(
524            parse_status("200, 204,300-301").unwrap(),
525            vec![(200, 200), (204, 204), (300, 301)]
526        );
527        for bad in ["", "abc", "399-200", "99", "200-700", "200,"] {
528            assert!(parse_status(bad).is_err(), "{bad}");
529        }
530    }
531
532    #[test]
533    fn words() {
534        assert_eq!(human(4_000), "4s");
535        assert_eq!(human(252_000), "4m 12s");
536        assert_eq!(human(7_500_000), "2h 5m");
537        assert_eq!(human(266_400_000), "3d 2h");
538        assert_eq!(Monitor::new("x", Kind::Tcp).interval, 60);
539    }
540}
isb_apps/monitor/mod.rs

isb_apps/monitor/
mod.rs
