Skip to main content

datumesh_http/
config.rs

1//! The mesh's configuration: [`Config`], from code or from the environment
2//! ([`Config::from_env`]).
3//!
4//! Secrets never come from here directly: the signing key and the API keys are read from
5//! files that must have mode 0600 ([`read_secret_file`]).
6
7use std::collections::BTreeMap;
8use std::fmt;
9use std::path::Path;
10use std::time::Duration;
11
12use datumesh_core::json::{self, Value};
13use datumesh_core::lexical;
14
15use crate::apikeys::ApiKeys;
16
17/// The default wall-clock budget of a request before it is answered `pending` (SPEC 5.5).
18pub const DEFAULT_BUDGET: Duration = Duration::from_secs(1);
19/// The default size of the blocking pool, which is the concurrency limit of evaluation and
20/// store access (see the crate documentation for why).
21pub const DEFAULT_POOL_SIZE: usize = 32;
22/// The default bound on a `POST /cells` batch.
23pub const DEFAULT_BATCH_MAX: usize = 1000;
24/// The default number of members in one JSON page of `GET /slice`.
25pub const DEFAULT_SLICE_PAGE: usize = 1000;
26/// The bound on a JSON page of `GET /slice` (`limit=`).
27pub const MAX_SLICE_PAGE: usize = 10_000;
28/// The bound on the members of a slice served (SPEC 10.1: a manifest has at most 1,048,576).
29pub const MAX_SLICE_MEMBERS: usize = 1_048_576;
30
31/// Rate limits (token buckets), per minute. Refused requests are answered `rate_limited`
32/// with `retry_after_seconds` and `Retry-After`; nothing is ever queued.
33#[non_exhaustive]
34#[derive(Clone, Debug, PartialEq, Eq)]
35pub struct RateLimits {
36    /// Whether limits apply at all (default true).
37    pub enabled: bool,
38    /// Requests per minute per client IP without an API key (default 60).
39    pub per_ip: u32,
40    /// The burst per IP: the bucket's capacity (default 120).
41    pub ip_burst: u32,
42    /// Requests per minute per API key, unless the key's own line says otherwise (default
43    /// 600; the burst is the same number).
44    pub per_key: u32,
45    /// `POST /verify` per minute per IP, with or without a key (default 60; burst the same).
46    pub verify_per_ip: u32,
47    /// Cells per request token in a `POST /cells` batch (default 50: a batch of 1,000 cells
48    /// counts as 20 requests).
49    pub batch_cells_per_token: u32,
50}
51
52impl Default for RateLimits {
53    fn default() -> Self {
54        RateLimits {
55            enabled: true,
56            per_ip: 60,
57            ip_burst: 120,
58            per_key: 600,
59            verify_per_ip: 60,
60            batch_cells_per_token: 50,
61        }
62    }
63}
64
65/// Cache lifetimes. Content-addressed objects (descriptors, receipts, manifests, blobs) never
66/// expire: their bytes are fixed by their hash.
67#[non_exhaustive]
68#[derive(Clone, Debug, PartialEq, Eq)]
69pub struct Ttls {
70    /// A foreign host's key document (default 300 s). A receipt signed with a key id the
71    /// cached document lacks refetches it at once (at most every `key_refetch`).
72    pub key_doc: Duration,
73    /// The shortest interval between two fetches of one host's key document (default 10 s).
74    pub key_refetch: Duration,
75    /// A foreign host's mesh document (default 300 s).
76    pub mesh_doc: Duration,
77    /// How long a foreign name's binding, confirmed with its mesh, stands for requests
78    /// without an asof or with a later one (default 60 s).
79    pub binding: Duration,
80    /// How long a foreign cell's latest version, confirmed with its mesh, stands for requests
81    /// without an asof or with a later one (default 60 s).
82    pub current: Duration,
83    /// `max-age` of answers to requests without an asof (default 10 s).
84    pub latest_max_age: Duration,
85    /// How long a completed job's result is kept after it is first served, and after it
86    /// completes when it is never served, while the mesh runs (default and minimum 300 s, SPEC 5.5).
87    pub job: Duration,
88}
89
90impl Default for Ttls {
91    fn default() -> Self {
92        Ttls {
93            key_doc: Duration::from_secs(300),
94            key_refetch: Duration::from_secs(10),
95            mesh_doc: Duration::from_secs(300),
96            binding: Duration::from_secs(60),
97            current: Duration::from_secs(60),
98            latest_max_age: Duration::from_secs(10),
99            job: Duration::from_secs(300),
100        }
101    }
102}
103
104/// What a function's cells need (the licence and payment stubs of step 1.4).
105#[non_exhaustive]
106#[derive(Clone, Debug, PartialEq, Eq)]
107pub enum Rule {
108    /// Only these principals (API key owners) may receive the function's cells; anyone
109    /// else is answered `unlicensed`.
110    Principals(Vec<String>),
111    /// The function's cells are sold: `payment_required` with this quote (a stub until
112    /// prepaid balances, 3.3).
113    Price {
114        /// The amount (SPEC 3.1, greater than zero).
115        amount: String,
116        /// A currency code of `units.md`.
117        currency: String,
118    },
119}
120
121/// Per-function rules, by fully qualified function name. Functions without one are free.
122#[non_exhaustive]
123#[derive(Clone, Debug, Default, PartialEq, Eq)]
124pub struct Policy {
125    /// The rules.
126    pub rules: BTreeMap<String, Rule>,
127}
128
129impl Policy {
130    /// The rule of a function, if any.
131    pub fn rule(&self, fn_name: &str) -> Option<&Rule> {
132        self.rules.get(fn_name)
133    }
134
135    /// From JSON: `{"<function>": {"principals": ["..."]} | {"price": {"amount", "currency"}}}`.
136    pub fn from_json(bytes: &[u8]) -> Result<Policy, ConfigError> {
137        let v = json::parse(bytes).map_err(|e| ConfigError::new("policy", e.to_string()))?;
138        let mut rules = BTreeMap::new();
139        for (name, r) in v.as_object().ok_or_else(|| ConfigError::new("policy", "not an object"))? {
140            lexical::check_function_name(name, "$").map_err(|e| ConfigError::new("policy", e.to_string()))?;
141            let rule = if let Some(Value::Array(ps)) = r.get("principals") {
142                Rule::Principals(
143                    ps.iter()
144                        .map(|p| p.as_str().map(str::to_string).ok_or_else(|| ConfigError::new("policy", "principal")))
145                        .collect::<Result<_, _>>()?,
146                )
147            } else if let Some(p) = r.get("price") {
148                let amount =
149                    p.get("amount").and_then(Value::as_str).ok_or_else(|| ConfigError::new("policy", "amount"))?;
150                let currency =
151                    p.get("currency").and_then(Value::as_str).ok_or_else(|| ConfigError::new("policy", "currency"))?;
152                lexical::check_number(amount, "$").map_err(|e| ConfigError::new("policy", e.to_string()))?;
153                if amount.starts_with('-') || amount == "0" || !lexical::is_currency(currency) {
154                    return Err(ConfigError::new("policy", format!("price of {name}")));
155                }
156                Rule::Price { amount: amount.to_string(), currency: currency.to_string() }
157            } else {
158                return Err(ConfigError::new("policy", format!("rule of {name}")));
159            };
160            rules.insert(name.clone(), rule);
161        }
162        Ok(Policy { rules })
163    }
164}
165
166/// The mesh's configuration.
167#[non_exhaustive]
168#[derive(Clone, Debug)]
169pub struct Config {
170    /// The DNS name the mesh signs as (`computed_by`) and serves on: `https://<host>/...`
171    /// (e.g. `mesh-test.datumesh.com`).
172    pub host: String,
173    /// The hosts whose names this mesh owns (default: `host`). A name under any other host
174    /// is resolved across meshes.
175    pub namespaces: Vec<String>,
176    /// The wall-clock budget of a request before it is answered `pending` (default 1 s).
177    pub budget: Duration,
178    /// The blocking pool's size: the concurrency limit (default 32).
179    pub pool_size: usize,
180    /// The most cells in one `POST /cells` (default 1,000).
181    pub batch_max: usize,
182    /// The default JSON page of `GET /slice` (default 1,000; at most 10,000 with `limit=`).
183    pub slice_page: usize,
184    /// The most members one slice read serves (default 1,048,576).
185    pub slice_max: usize,
186    /// The depth of `GET /explain` by default (default 4; at most 16 with `depth=`).
187    pub explain_depth: usize,
188    /// The most jobs held at once, running or done (default 10,000).
189    pub max_jobs: usize,
190    /// Rate limits.
191    pub rate: RateLimits,
192    /// Cache lifetimes.
193    pub ttl: Ttls,
194    /// Take the client IP from the last `X-Forwarded-For` entry (behind Caddy) instead of
195    /// the peer address (default false).
196    pub trust_proxy: bool,
197    /// Resolve names under hosts this mesh does not own (default true).
198    pub cross_mesh: bool,
199    /// When set, the only other hosts whose names are resolved across meshes; names under
200    /// any other host are not served (`not_covered`). Default: every host.
201    pub cross_mesh_hosts: Option<Vec<String>>,
202    /// Origins to use instead of `https://<host>` for some hosts, e.g. `mesh-b.example` ->
203    /// `http://127.0.0.1:8081` (tests and private networks only; the default is none).
204    pub peers: BTreeMap<String, String>,
205    /// The time limit of one request to another mesh (default 10 s; SPEC 9.3's bound).
206    pub peer_timeout: Duration,
207    /// Short names of the call syntax: `@alias` -> fully qualified name.
208    pub aliases: BTreeMap<String, String>,
209    /// Licence and payment rules.
210    pub policy: Policy,
211    /// Descriptions of the functions `GET /find` searches (name -> description).
212    pub catalog: BTreeMap<String, String>,
213    /// API keys (`Authorization: Bearer dm_...`).
214    pub api_keys: ApiKeys,
215}
216
217impl Config {
218    /// The defaults for a mesh signing as `host`.
219    pub fn new(host: impl Into<String>) -> Config {
220        let host = host.into();
221        Config {
222            namespaces: vec![host.clone()],
223            host,
224            budget: DEFAULT_BUDGET,
225            pool_size: DEFAULT_POOL_SIZE,
226            batch_max: DEFAULT_BATCH_MAX,
227            slice_page: DEFAULT_SLICE_PAGE,
228            slice_max: MAX_SLICE_MEMBERS,
229            explain_depth: 4,
230            max_jobs: 10_000,
231            rate: RateLimits::default(),
232            ttl: Ttls::default(),
233            trust_proxy: false,
234            cross_mesh: true,
235            cross_mesh_hosts: None,
236            peers: BTreeMap::new(),
237            peer_timeout: Duration::from_secs(10),
238            aliases: BTreeMap::new(),
239            policy: Policy::default(),
240            catalog: BTreeMap::new(),
241            api_keys: ApiKeys::default(),
242        }
243    }
244
245    /// The configuration from the environment (`DATUMESH_*`; the README lists them). The
246    /// files named (`DATUMESH_API_KEYS_FILE`, and the others) are read here; the API keys
247    /// file must have mode 0600.
248    pub fn from_env() -> Result<Config, ConfigError> {
249        Config::from_vars(|k| std::env::var(k).ok())
250    }
251
252    /// The configuration from variables `get` answers, as [`Config::from_env`] reads them.
253    pub fn from_vars(get: impl Fn(&str) -> Option<String>) -> Result<Config, ConfigError> {
254        let host = get("DATUMESH_HOST").ok_or_else(|| ConfigError::new("DATUMESH_HOST", "not set"))?;
255        let mut c = Config::new(host);
256        if let Some(v) = get("DATUMESH_NAMESPACES") {
257            c.namespaces = v.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
258        }
259        let ms = |k: &str, d: Duration| -> Result<Duration, ConfigError> {
260            match get(k) {
261                Some(v) => v.parse::<u64>().map(Duration::from_millis).map_err(|e| ConfigError::new(k, e.to_string())),
262                None => Ok(d),
263            }
264        };
265        let secs = |k: &str, d: Duration| -> Result<Duration, ConfigError> {
266            match get(k) {
267                Some(v) => v.parse::<u64>().map(Duration::from_secs).map_err(|e| ConfigError::new(k, e.to_string())),
268                None => Ok(d),
269            }
270        };
271        let num = |k: &str, d: usize| -> Result<usize, ConfigError> {
272            match get(k) {
273                Some(v) => v.parse::<usize>().map_err(|e| ConfigError::new(k, e.to_string())),
274                None => Ok(d),
275            }
276        };
277        let flag = |k: &str, d: bool| -> Result<bool, ConfigError> {
278            match get(k).as_deref() {
279                None => Ok(d),
280                Some("1" | "true" | "yes") => Ok(true),
281                Some("0" | "false" | "no") => Ok(false),
282                Some(v) => Err(ConfigError::new(k, format!("not a boolean: {v}"))),
283            }
284        };
285        c.budget = ms("DATUMESH_BUDGET_MS", c.budget)?;
286        c.pool_size = num("DATUMESH_POOL_SIZE", c.pool_size)?;
287        c.batch_max = num("DATUMESH_BATCH_MAX", c.batch_max)?;
288        c.slice_page = num("DATUMESH_SLICE_PAGE", c.slice_page)?;
289        c.slice_max = num("DATUMESH_SLICE_MAX", c.slice_max)?;
290        c.explain_depth = num("DATUMESH_EXPLAIN_DEPTH", c.explain_depth)?;
291        c.max_jobs = num("DATUMESH_MAX_JOBS", c.max_jobs)?;
292        c.trust_proxy = flag("DATUMESH_TRUST_PROXY", c.trust_proxy)?;
293        c.cross_mesh = flag("DATUMESH_CROSS_MESH", c.cross_mesh)?;
294        c.rate.enabled = flag("DATUMESH_RATE_LIMITS", c.rate.enabled)?;
295        c.rate.per_ip = num("DATUMESH_RATE_PER_IP", c.rate.per_ip as usize)? as u32;
296        c.rate.ip_burst = num("DATUMESH_RATE_IP_BURST", c.rate.ip_burst as usize)? as u32;
297        c.rate.per_key = num("DATUMESH_RATE_PER_KEY", c.rate.per_key as usize)? as u32;
298        c.rate.verify_per_ip = num("DATUMESH_RATE_VERIFY_PER_IP", c.rate.verify_per_ip as usize)? as u32;
299        c.ttl.key_doc = secs("DATUMESH_TTL_KEY_DOC", c.ttl.key_doc)?;
300        c.ttl.mesh_doc = secs("DATUMESH_TTL_MESH_DOC", c.ttl.mesh_doc)?;
301        c.ttl.binding = secs("DATUMESH_TTL_BINDING", c.ttl.binding)?;
302        c.ttl.current = secs("DATUMESH_TTL_CURRENT", c.ttl.current)?;
303        c.ttl.latest_max_age = secs("DATUMESH_LATEST_MAX_AGE", c.ttl.latest_max_age)?;
304        c.ttl.job = secs("DATUMESH_JOB_TTL", c.ttl.job)?;
305        c.peer_timeout = ms("DATUMESH_PEER_TIMEOUT_MS", c.peer_timeout)?;
306        if let Some(v) = get("DATUMESH_CROSS_MESH_HOSTS") {
307            c.cross_mesh_hosts = Some(v.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect());
308        }
309        if let Some(v) = get("DATUMESH_PEERS") {
310            for pair in v.split(',').map(str::trim).filter(|s| !s.is_empty()) {
311                let (h, o) =
312                    pair.split_once('=').ok_or_else(|| ConfigError::new("DATUMESH_PEERS", "host=origin pairs"))?;
313                c.peers.insert(h.trim().to_string(), o.trim().trim_end_matches('/').to_string());
314            }
315        }
316        if let Some(p) = get("DATUMESH_ALIASES_FILE") {
317            c.aliases = string_map(&read_file(&p)?, "DATUMESH_ALIASES_FILE")?;
318        }
319        if let Some(p) = get("DATUMESH_CATALOG_FILE") {
320            c.catalog = string_map(&read_file(&p)?, "DATUMESH_CATALOG_FILE")?;
321        }
322        if let Some(p) = get("DATUMESH_POLICY_FILE") {
323            c.policy = Policy::from_json(&read_file(&p)?)?;
324        }
325        if let Some(p) = get("DATUMESH_API_KEYS_FILE") {
326            c.api_keys = ApiKeys::parse(&read_secret_file(Path::new(&p))?)?;
327        }
328        c.check()?;
329        Ok(c)
330    }
331
332    /// Checks the configuration: names are DNS names and function names, numbers in range.
333    pub fn check(&self) -> Result<(), ConfigError> {
334        lexical::check_dns_name(&self.host, "$").map_err(|e| ConfigError::new("host", e.to_string()))?;
335        for h in &self.namespaces {
336            lexical::check_dns_name(h, "$").map_err(|e| ConfigError::new("namespaces", e.to_string()))?;
337        }
338        for (h, o) in &self.peers {
339            lexical::check_dns_name(h, "$").map_err(|e| ConfigError::new("peers", e.to_string()))?;
340            if !(o.starts_with("http://") || o.starts_with("https://")) {
341                return Err(ConfigError::new("peers", format!("{o}: not an http(s) origin")));
342            }
343        }
344        for (a, n) in &self.aliases {
345            lexical::check_function_name(n, "$").map_err(|e| ConfigError::new("aliases", format!("{a}: {e}")))?;
346        }
347        if self.pool_size == 0 || self.batch_max == 0 || self.slice_page == 0 || self.slice_max == 0 {
348            return Err(ConfigError::new("sizes", "the pool, batch and slice bounds must be positive"));
349        }
350        if self.ttl.job < Duration::from_secs(300) {
351            return Err(ConfigError::new("job TTL", "at least 300 s (SPEC 5.5)"));
352        }
353        if self.budget.is_zero() {
354            return Err(ConfigError::new("budget", "must be positive"));
355        }
356        Ok(())
357    }
358
359    /// Whether this mesh owns the names under `host`.
360    pub fn owns(&self, host: &str) -> bool {
361        self.namespaces.iter().any(|h| h == host)
362    }
363
364    /// `https://<host>`: where this mesh is served.
365    pub fn origin(&self) -> String {
366        format!("https://{}", self.host)
367    }
368}
369
370fn string_map(bytes: &[u8], what: &str) -> Result<BTreeMap<String, String>, ConfigError> {
371    let v = json::parse(bytes).map_err(|e| ConfigError::new(what, e.to_string()))?;
372    let mut out = BTreeMap::new();
373    for (k, x) in v.as_object().ok_or_else(|| ConfigError::new(what, "not an object"))? {
374        let s = x.as_str().ok_or_else(|| ConfigError::new(what, format!("{k}: not a string")))?;
375        out.insert(k.clone(), s.to_string());
376    }
377    Ok(out)
378}
379
380fn read_file(path: &str) -> Result<Vec<u8>, ConfigError> {
381    std::fs::read(path).map_err(|e| ConfigError::new(path, e.to_string()))
382}
383
384/// Reads a file holding a secret, refusing it unless only its owner may read or write it
385/// (mode 0600 or stricter, on Unix).
386pub fn read_secret_file(path: &Path) -> Result<Vec<u8>, ConfigError> {
387    let what = path.display().to_string();
388    let meta = std::fs::metadata(path).map_err(|e| ConfigError::new(&what, e.to_string()))?;
389    #[cfg(unix)]
390    {
391        use std::os::unix::fs::PermissionsExt;
392        let mode = meta.permissions().mode() & 0o777;
393        if mode & 0o077 != 0 {
394            return Err(ConfigError::new(&what, format!("mode {mode:o}: a secret file must be 0600")));
395        }
396    }
397    #[cfg(not(unix))]
398    let _ = meta;
399    std::fs::read(path).map_err(|e| ConfigError::new(&what, e.to_string()))
400}
401
402/// A configuration error: what and why.
403#[derive(Clone, Debug, PartialEq, Eq)]
404pub struct ConfigError {
405    what: String,
406    detail: String,
407}
408
409impl ConfigError {
410    pub(crate) fn new(what: &str, detail: impl Into<String>) -> ConfigError {
411        ConfigError { what: what.to_string(), detail: detail.into() }
412    }
413}
414
415impl fmt::Display for ConfigError {
416    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417        write!(f, "configuration: {}: {}", self.what, self.detail)
418    }
419}
420
421impl std::error::Error for ConfigError {}