isb_core/org/
limits.rs

1//! An org's quotas: what is allocated against them, and incus' refusals
2//! said in isb's terms.
3//!
4//! An org's `limits.cpu`, `limits.memory` and `limits.disk` are incus
5//! project limits, and incus enforces them as budgets: the sum of what every
6//! instance in the project is configured with (its `limits.cpu`, its
7//! `limits.memory`, its root disk's `size`, and the volumes' sizes), stopped
8//! instances included, against the limit. Nothing measures actual use. A
9//! shared ceiling on what the org's instances use together would need a
10//! parent cgroup per project, which incus does not offer.
11//!
12//! incus checks the budgets when an instance or a volume is created or
13//! resized, and refuses with text such as `Reached maximum aggregate value
14//! "2" for "limits.cpu" in project "isb-lab"`. The client turns any such
15//! answer into [`Error::Invalid`] with [`explain`]'s message, so every path
16//! that creates instances in an org (sandboxes, workspaces, apps, databases,
17//! builds, stacks) says the same thing.
18
19use std::collections::BTreeMap;
20
21use serde::{Deserialize, Serialize};
22use serde_json::Value;
23
24use super::OrgId;
25use crate::client::Client;
26use crate::error::Error;
27
28/// The root disk size isb gives an instance it creates in an org with
29/// `limits.disk` when its spec sets none (incus refuses an instance without
30/// one there); see [`super::disk`].
31pub const DEFAULT_ROOT_SIZE: &str = "10GiB";
32
33/// One of an org's limits, which `isb org update` can lift again.
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "lowercase")]
36pub enum Limit {
37    Cpus,
38    Memory,
39    Disk,
40    Instances,
41}
42
43impl Limit {
44    /// The incus project key.
45    pub fn key(self) -> &'static str {
46        match self {
47            Limit::Cpus => "limits.cpu",
48            Limit::Memory => "limits.memory",
49            Limit::Disk => "limits.disk",
50            Limit::Instances => "limits.instances",
51        }
52    }
53}
54
55/// One limit of an org: the limit, what its instances are allocated against
56/// it (summed over every instance, stopped ones included), and what is left.
57/// Bytes for memory and disk.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
59pub struct Budget {
60    pub limit: i64,
61    pub allocated: i64,
62    pub free: i64,
63}
64
65impl Budget {
66    /// `3 of 4 allocated, 1 free`, sizes as incus writes them.
67    pub fn text(&self, in_bytes: bool) -> String {
68        let f = |n: i64| if in_bytes { bytes(n) } else { n.to_string() };
69        format!(
70            "{} of {} allocated, {} free",
71            f(self.allocated),
72            f(self.limit),
73            f(self.free)
74        )
75    }
76}
77
78/// The org's limited resources (`cpu`, `memory`, `disk`, `instances`) as
79/// budgets, from `/1.0/projects/<p>/state`. A resource without a limit is
80/// left out: incus does not total what nothing limits.
81pub(crate) fn budgets(state: &Value) -> BTreeMap<String, Budget> {
82    ["cpu", "memory", "disk", "instances"]
83        .into_iter()
84        .filter_map(|name| {
85            let (limit, allocated) = usage(state, &format!("limits.{name}"))?;
86            (limit >= 0).then(|| {
87                let b = Budget {
88                    limit,
89                    allocated,
90                    free: (limit - allocated).max(0),
91                };
92                (name.to_string(), b)
93            })
94        })
95        .collect()
96}
97
98/// A project's budgets, read from incus; empty when its state can't be read.
99pub(crate) fn read_budgets(c: &Client, project: &str) -> BTreeMap<String, Budget> {
100    c.clone()
101        .project("")
102        .get(&format!(
103            "/1.0/projects/{}/state",
104            crate::client::encode_segment(project)
105        ))
106        .map(|s| budgets(&s))
107        .unwrap_or_default()
108}
109
110/// A refusal of one of a project's limits.
111#[derive(Debug, Clone, PartialEq, Eq)]
112pub(crate) struct Refusal {
113    /// The limit's key, e.g. `limits.cpu`.
114    pub key: String,
115    /// Its value as incus quoted it, when it did.
116    pub value: Option<String>,
117    pub project: String,
118}
119
120/// The project-limit refusal in an incus error message, if it is one.
121pub(crate) fn parse(msg: &str) -> Option<Refusal> {
122    let at = msg.find("Reached maximum ")?;
123    let rest = &msg[at..];
124    let quoted: Vec<&str> = rest.split('"').skip(1).step_by(2).collect();
125    if rest.starts_with("Reached maximum aggregate value ") {
126        let [value, key, project] = quoted[..] else {
127            return None;
128        };
129        return Some(Refusal {
130            key: key.to_string(),
131            value: Some(value.to_string()),
132            project: project.to_string(),
133        });
134    }
135    if rest.starts_with("Reached maximum number of instances of type ") {
136        let [kind, project] = quoted[..] else {
137            return None;
138        };
139        let key = if kind.starts_with("virtual") {
140            "limits.virtual-machines"
141        } else {
142            "limits.containers"
143        };
144        return Some(Refusal {
145            key: key.into(),
146            value: None,
147            project: project.to_string(),
148        });
149    }
150    if rest.starts_with("Reached maximum number of instances in project ") {
151        let [project] = quoted[..] else { return None };
152        return Some(Refusal {
153            key: "limits.instances".into(),
154            value: None,
155            project: project.to_string(),
156        });
157    }
158    None
159}
160
161/// The project's limit and allocation of what `key` limits, from
162/// `/1.0/projects/<p>/state` (`resources.<name>.{Limit,Usage}`; incus'
163/// "usage" is the sum of the instances' configured limits).
164pub(crate) fn usage(state: &Value, key: &str) -> Option<(i64, i64)> {
165    let name = key.strip_prefix("limits.")?;
166    let r = &state["resources"][name];
167    let get = |k: &str, alt: &str| r[k].as_i64().or_else(|| r[alt].as_i64());
168    Some((get("Limit", "limit")?, get("Usage", "usage")?))
169}
170
171/// Bytes as incus sizes are written: `3.5GiB`, `512MiB`.
172pub fn bytes(n: i64) -> String {
173    const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
174    let mut v = n as f64;
175    let mut u = 0;
176    while v >= 1024.0 && u < UNITS.len() - 1 {
177        v /= 1024.0;
178        u += 1;
179    }
180    if v.fract() == 0.0 {
181        format!("{v}{}", UNITS[u])
182    } else {
183        format!("{v:.1}{}", UNITS[u])
184    }
185}
186
187/// What the limit counts, and `isb org update`'s flag that sets it.
188fn describe(key: &str) -> (&'static str, Option<&'static str>) {
189    match key {
190        "limits.cpu" => ("CPU", Some("--cpus")),
191        "limits.memory" => ("memory", Some("--memory")),
192        k if k.starts_with("limits.disk") => ("disk", Some("--disk")),
193        "limits.instances" => ("instance", Some("--instances")),
194        "limits.containers" => ("container", None),
195        "limits.virtual-machines" => ("VM", None),
196        _ => ("resource", None),
197    }
198}
199
200/// What the refused request asked for: the instance (or volume) and its
201/// amount of the limited resource.
202#[derive(Debug, Clone, PartialEq, Eq)]
203pub(crate) struct Need {
204    pub name: Option<String>,
205    pub amount: String,
206}
207
208/// What a refused create or update of an instance or volume (`path`,
209/// `body`) asked for of `key`: the body's own value, else the org's
210/// default profile's (what the instance would have got).
211fn needed(c: &Client, key: &str, path: &str, body: &Value) -> Option<Need> {
212    let tail = path.split('?').next().unwrap_or(path);
213    let name = body["name"]
214        .as_str()
215        .map(String::from)
216        .or_else(|| tail.rsplit('/').next().map(String::from))
217        .filter(|n| !n.is_empty() && n != "instances" && n != "volumes");
218    let disk = key.starts_with("limits.disk");
219    let amount = if tail.contains("/volumes") {
220        disk.then(|| body["config"]["size"].as_str().map(String::from))??
221    } else if tail.starts_with("/1.0/instances") {
222        let own = if disk {
223            body["devices"]["root"]["size"].as_str()
224        } else {
225            body["config"][key].as_str()
226        };
227        match own {
228            Some(v) => v.to_string(),
229            None if key == "limits.cpu" || key == "limits.memory" || disk => {
230                let p = c.get("/1.0/profiles/default").ok()?;
231                let v = if disk {
232                    &p["devices"]["root"]["size"]
233                } else {
234                    &p["config"][key]
235                };
236                format!("{} (the org's default)", v.as_str()?)
237            }
238            None => return None,
239        }
240    } else {
241        return None;
242    };
243    Some(Need { name, amount })
244}
245
246/// The message for a refusal, given the project's limit and allocation when
247/// they could be read, and what the request needed when it is known.
248pub(crate) fn explain(r: &Refusal, used: Option<(i64, i64)>, need: Option<&Need>) -> String {
249    let (what, flag) = describe(&r.key);
250    let in_bytes = matches!(what, "memory" | "disk");
251    let fmt = |n: i64| if in_bytes { bytes(n) } else { n.to_string() };
252    let counted = matches!(what, "instance" | "container" | "VM");
253    let state = match used {
254        Some((l, u)) if counted => format!("{}: {u} of {l}, stopped ones included", r.key),
255        Some((l, u)) => format!(
256            "{}: allocated {} of {}, the sum of every instance's limit, stopped ones included",
257            r.key,
258            fmt(u),
259            fmt(l)
260        ),
261        None => format!("{} {}", r.key, r.value.as_deref().unwrap_or("?")),
262    };
263    let need = need
264        .map(|n| match &n.name {
265            Some(name) => format!("; {name} needs {}", n.amount),
266            None => format!("; the request needs {}", n.amount),
267        })
268        .unwrap_or_default();
269    let free = if counted {
270        "delete one, or ask for fewer"
271    } else {
272        "delete an instance or lower one's limit (stopping one frees nothing), or ask for less"
273    };
274    match OrgId::from_incus_project(&r.project) {
275        Some(org) => {
276            let raise = match flag {
277                Some(f) => format!(
278                    "a platform admin raises it with `isb org update {org} {f} N` on the host, or lifts it with `{f} none` (or the org_update tool)"
279                ),
280                None => format!(
281                    "isb does not set {}; an operator raises it with `incus project set {} {}=N`",
282                    r.key, r.project, r.key
283                ),
284            };
285            format!("org {org} is at its {what} quota ({state}{need}): {free}, or {raise}")
286        }
287        None => format!(
288            "incus project {} is at its {what} limit ({state}{need}): {free}, or raise it with `incus project set {} {}=...`",
289            r.project, r.project, r.key
290        ),
291    }
292}
293
294/// `message` from incusd, as the error to return: a clear quota error when
295/// it is a project-limit refusal (with the allocation `c` can read and what
296/// `request`, the refused path and body, asked for), else `None`.
297pub(crate) fn translate(
298    c: &Client,
299    message: &str,
300    request: Option<(&str, &Value)>,
301) -> Option<Error> {
302    let r = parse(message)?;
303    let state = c.clone().project("").get(&format!(
304        "/1.0/projects/{}/state",
305        crate::client::encode_segment(&r.project)
306    ));
307    let used = state.ok().and_then(|s| usage(&s, &r.key));
308    let need = request.and_then(|(path, body)| needed(c, &r.key, path, body));
309    Some(Error::Invalid(explain(&r, used, need.as_ref())))
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315    use serde_json::json;
316
317    #[test]
318    fn the_client_turns_a_quota_refusal_into_a_clear_error() {
319        use crate::client::fake::{Route, serve};
320        let (_d, c) = serve(vec![
321            Route {
322                prefix: "POST /1.0/instances",
323                status: 500,
324                body: json!(
325                    r#"Failed checking if instance creation allowed: Reached maximum aggregate value "2" for "limits.cpu" in project "isb-lab""#
326                ),
327            },
328            Route {
329                prefix: "GET /1.0/projects/isb-lab/state",
330                status: 200,
331                body: json!({"resources": {"cpu": {"Limit": 2, "Usage": 2}}}),
332            },
333            Route {
334                prefix: "GET /1.0/profiles/default",
335                status: 200,
336                body: json!({"config": {"limits.cpu": "1"}, "devices": {}}),
337            },
338        ]);
339        let c = c.project("isb-lab");
340        let e = c
341            .mutate(
342                "POST",
343                "/1.0/instances",
344                Some(&json!({"name": "web-1", "config": {}})),
345                "create",
346                c.timeouts.other,
347            )
348            .unwrap_err();
349        assert!(matches!(e, Error::Invalid(_)), "{e:?}");
350        assert!(
351            e.to_string().starts_with(
352                "org lab is at its CPU quota (limits.cpu: allocated 2 of 2, the sum of every instance's limit, stopped ones included; web-1 needs 1 (the org's default))"
353            ),
354            "{e}"
355        );
356        // Any other incus error is passed on as it was.
357        let e = c.get("/1.0/instances/x").unwrap_err();
358        assert!(e.is_not_found(), "{e}");
359    }
360
361    #[test]
362    fn parses_incus_refusals() {
363        let m = r#"Failed checking if instance creation allowed: Reached maximum aggregate value "2" for "limits.cpu" in project "isb-lab""#;
364        assert_eq!(
365            parse(m),
366            Some(Refusal {
367                key: "limits.cpu".into(),
368                value: Some("2".into()),
369                project: "isb-lab".into()
370            })
371        );
372        let m = r#"Failed checking if instance creation allowed: Reached maximum number of instances of type "container" in project "isb-lab""#;
373        assert_eq!(parse(m).unwrap().key, "limits.containers");
374        let m = r#"Reached maximum number of instances in project "isb-lab""#;
375        let r = parse(m).unwrap();
376        assert_eq!((r.key.as_str(), r.value), ("limits.instances", None));
377        assert_eq!(parse("Project not found"), None);
378        assert_eq!(parse(r#"Reached maximum aggregate value "2""#), None);
379    }
380
381    #[test]
382    fn reads_usage_from_project_state() {
383        let s = json!({"resources": {"cpu": {"Limit": 2, "Usage": 2}, "memory": {"limit": 4, "usage": 3}}});
384        assert_eq!(usage(&s, "limits.cpu"), Some((2, 2)));
385        assert_eq!(usage(&s, "limits.memory"), Some((4, 3)));
386        assert_eq!(usage(&s, "limits.disk"), None);
387    }
388
389    #[test]
390    fn budgets_are_the_limited_resources_with_what_is_left() {
391        let s = json!({"resources": {
392            "cpu": {"Limit": 4, "Usage": 3},
393            "memory": {"Limit": 4i64 << 30, "Usage": 5i64 << 30},
394            "disk": {"Limit": -1, "Usage": 0},
395            "instances": {"Limit": 5, "Usage": 3},
396            "networks": {"Limit": 2, "Usage": 0},
397        }});
398        let b = budgets(&s);
399        assert_eq!(b.keys().collect::<Vec<_>>(), ["cpu", "instances", "memory"]);
400        assert_eq!(
401            b["cpu"],
402            Budget {
403                limit: 4,
404                allocated: 3,
405                free: 1
406            }
407        );
408        assert_eq!(b["cpu"].text(false), "3 of 4 allocated, 1 free");
409        // Over budget (the limit was lowered under what is allocated): none free.
410        assert_eq!(b["memory"].free, 0);
411        assert_eq!(b["memory"].text(true), "5GiB of 4GiB allocated, 0B free");
412    }
413
414    #[test]
415    fn a_refused_request_says_what_it_needed() {
416        let (_d, c) = crate::client::fake::serve(vec![]);
417        let body = json!({"name": "db-1", "config": {"limits.memory": "2GiB"}, "devices": {"root": {"size": "20GiB"}}});
418        let n = needed(&c, "limits.memory", "/1.0/instances", &body).unwrap();
419        assert_eq!(
420            (n.name.as_deref(), n.amount.as_str()),
421            (Some("db-1"), "2GiB")
422        );
423        let n = needed(&c, "limits.disk", "/1.0/instances", &body).unwrap();
424        assert_eq!(n.amount, "20GiB");
425        // A resize names the instance from the path.
426        let n = needed(
427            &c,
428            "limits.cpu",
429            "/1.0/instances/web-2",
430            &json!({"config": {"limits.cpu": "4"}}),
431        )
432        .unwrap();
433        assert_eq!((n.name.as_deref(), n.amount.as_str()), (Some("web-2"), "4"));
434        let v = json!({"name": "data", "config": {"size": "5GiB"}});
435        let n = needed(&c, "limits.disk", "/1.0/storage-pools/p/volumes/custom", &v).unwrap();
436        assert_eq!(
437            (n.name.as_deref(), n.amount.as_str()),
438            (Some("data"), "5GiB")
439        );
440        // Counts, and requests that are not about an instance or a volume: nothing.
441        assert_eq!(
442            needed(&c, "limits.instances", "/1.0/instances", &body),
443            None
444        );
445        assert_eq!(
446            needed(&c, "limits.cpu", "/1.0/projects/isb-lab", &body),
447            None
448        );
449    }
450
451    #[test]
452    fn explains_a_full_quota_with_its_allocation_and_the_way_to_raise_it() {
453        let r = Refusal {
454            key: "limits.cpu".into(),
455            value: Some("2".into()),
456            project: "isb-lab".into(),
457        };
458        let m = explain(&r, Some((2, 2)), None);
459        assert!(
460            m.starts_with("org lab is at its CPU quota (limits.cpu: allocated 2 of 2, the sum of every instance's limit, stopped ones included):"),
461            "{m}"
462        );
463        assert!(m.contains("stopping one frees nothing"), "{m}");
464        assert!(m.contains("`isb org update lab --cpus N`"), "{m}");
465        assert!(m.contains("`--cpus none`"), "{m}");
466        assert!(m.contains("org_update"), "{m}");
467
468        let r = Refusal {
469            key: "limits.memory".into(),
470            value: None,
471            project: "isb-lab".into(),
472        };
473        let need = Need {
474            name: Some("db-1".into()),
475            amount: "1GiB".into(),
476        };
477        let m = explain(&r, Some((4 << 30, 3584 << 20)), Some(&need));
478        assert!(
479            m.contains("(limits.memory: allocated 3.5GiB of 4GiB, the sum of every instance's limit, stopped ones included; db-1 needs 1GiB)"),
480            "{m}"
481        );
482        assert!(m.contains("--memory N"), "{m}");
483
484        let r = Refusal {
485            key: "limits.instances".into(),
486            value: None,
487            project: "isb-lab".into(),
488        };
489        let m = explain(&r, Some((3, 3)), None);
490        assert!(
491            m.contains("(limits.instances: 3 of 3, stopped ones included): delete one"),
492            "{m}"
493        );
494
495        let r = Refusal {
496            key: "limits.containers".into(),
497            value: None,
498            project: "isb-lab".into(),
499        };
500        let m = explain(&r, None, None);
501        assert!(m.contains("(limits.containers ?)"), "{m}");
502        assert!(
503            m.contains("incus project set isb-lab limits.containers=N"),
504            "{m}"
505        );
506
507        let r = Refusal {
508            key: "limits.cpu".into(),
509            value: Some("8".into()),
510            project: "someone-else".into(),
511        };
512        let m = explain(&r, None, None);
513        assert!(
514            m.starts_with("incus project someone-else is at its CPU limit (limits.cpu 8)"),
515            "{m}"
516        );
517    }
518}
isb_core/org/limits.rs

isb_core/org/
limits.rs
