use std::collections::BTreeMap;
use std::fmt;
use std::path::Path;
use std::time::Duration;
use datumesh_core::json::{self, Value};
use datumesh_core::lexical;
use crate::apikeys::ApiKeys;
pub const DEFAULT_BUDGET: Duration = Duration::from_secs(1);
pub const DEFAULT_POOL_SIZE: usize = 32;
pub const DEFAULT_BATCH_MAX: usize = 1000;
pub const DEFAULT_SLICE_PAGE: usize = 1000;
pub const MAX_SLICE_PAGE: usize = 10_000;
pub const MAX_SLICE_MEMBERS: usize = 1_048_576;
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RateLimits {
pub enabled: bool,
pub per_ip: u32,
pub ip_burst: u32,
pub per_key: u32,
pub verify_per_ip: u32,
pub calc_per_ip: u32,
pub batch_cells_per_token: u32,
pub series_per_ip: u32,
}
impl Default for RateLimits {
fn default() -> Self {
RateLimits {
enabled: true,
per_ip: 60,
ip_burst: 120,
per_key: 600,
verify_per_ip: 60,
calc_per_ip: 30,
batch_cells_per_token: 50,
series_per_ip: 30,
}
}
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Ttls {
pub key_doc: Duration,
pub key_refetch: Duration,
pub mesh_doc: Duration,
pub binding: Duration,
pub current: Duration,
pub latest_max_age: Duration,
pub job: Duration,
}
impl Default for Ttls {
fn default() -> Self {
Ttls {
key_doc: Duration::from_secs(300),
key_refetch: Duration::from_secs(10),
mesh_doc: Duration::from_secs(300),
binding: Duration::from_secs(60),
current: Duration::from_secs(60),
latest_max_age: Duration::from_secs(10),
job: Duration::from_secs(300),
}
}
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Rule {
Principals(Vec<String>),
Price {
amount: String,
currency: String,
},
}
#[non_exhaustive]
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Policy {
pub rules: BTreeMap<String, Rule>,
}
impl Policy {
pub fn rule(&self, fn_name: &str) -> Option<&Rule> {
self.rules.get(fn_name)
}
pub fn from_json(bytes: &[u8]) -> Result<Policy, ConfigError> {
let v = json::parse(bytes).map_err(|e| ConfigError::new("policy", e.to_string()))?;
let mut rules = BTreeMap::new();
for (name, r) in v.as_object().ok_or_else(|| ConfigError::new("policy", "not an object"))? {
lexical::check_function_name(name, "$").map_err(|e| ConfigError::new("policy", e.to_string()))?;
let rule = if let Some(Value::Array(ps)) = r.get("principals") {
Rule::Principals(
ps.iter()
.map(|p| p.as_str().map(str::to_string).ok_or_else(|| ConfigError::new("policy", "principal")))
.collect::<Result<_, _>>()?,
)
} else if let Some(p) = r.get("price") {
let amount =
p.get("amount").and_then(Value::as_str).ok_or_else(|| ConfigError::new("policy", "amount"))?;
let currency =
p.get("currency").and_then(Value::as_str).ok_or_else(|| ConfigError::new("policy", "currency"))?;
lexical::check_number(amount, "$").map_err(|e| ConfigError::new("policy", e.to_string()))?;
if amount.starts_with('-') || amount == "0" || !lexical::is_currency(currency) {
return Err(ConfigError::new("policy", format!("price of {name}")));
}
Rule::Price { amount: amount.to_string(), currency: currency.to_string() }
} else {
return Err(ConfigError::new("policy", format!("rule of {name}")));
};
rules.insert(name.clone(), rule);
}
Ok(Policy { rules })
}
}
#[non_exhaustive]
#[derive(Clone, Debug)]
pub struct Config {
pub host: String,
pub base_url: Option<String>,
pub namespaces: Vec<String>,
pub budget: Duration,
pub pool_size: usize,
pub batch_max: usize,
pub test_cells: bool,
pub slice_page: usize,
pub slice_max: usize,
pub explain_depth: usize,
pub max_jobs: usize,
pub rate: RateLimits,
pub ttl: Ttls,
pub trust_proxy: bool,
pub cross_mesh: bool,
pub cross_mesh_hosts: Option<Vec<String>>,
pub peers: BTreeMap<String, String>,
pub peer_timeout: Duration,
pub aliases: BTreeMap<String, String>,
pub policy: Policy,
pub catalog: BTreeMap<String, String>,
pub embed_url: Option<String>,
pub api_keys: ApiKeys,
pub publish_domains: PublishDomains,
pub log_dir: Option<std::path::PathBuf>,
pub(crate) series: Vec<crate::series::SeriesPage>,
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub enum PublishDomains {
#[default]
Own,
Hosts(Vec<String>),
Any,
}
impl PublishDomains {
pub fn parse(v: &str) -> Result<PublishDomains, ConfigError> {
let v = v.trim();
if v.is_empty() {
return Ok(PublishDomains::Own);
}
if v == "*" {
return Ok(PublishDomains::Any);
}
let mut hosts = Vec::new();
for h in v.split(',').map(str::trim).filter(|h| !h.is_empty()) {
lexical::check_dns_name(h, "$").map_err(|e| ConfigError::new("DATUMESH_PUBLISH_DOMAINS", e.to_string()))?;
hosts.push(h.to_string());
}
Ok(if hosts.is_empty() { PublishDomains::Own } else { PublishDomains::Hosts(hosts) })
}
pub fn allows(&self, host: &str) -> bool {
match self {
PublishDomains::Own => false,
PublishDomains::Hosts(h) => h.iter().any(|x| x == host),
PublishDomains::Any => true,
}
}
}
impl Config {
pub fn new(host: impl Into<String>) -> Config {
let host = host.into();
Config {
namespaces: vec![host.clone()],
host,
base_url: None,
budget: DEFAULT_BUDGET,
pool_size: DEFAULT_POOL_SIZE,
batch_max: DEFAULT_BATCH_MAX,
test_cells: false,
slice_page: DEFAULT_SLICE_PAGE,
slice_max: MAX_SLICE_MEMBERS,
explain_depth: 4,
max_jobs: 10_000,
rate: RateLimits::default(),
ttl: Ttls::default(),
trust_proxy: false,
cross_mesh: true,
cross_mesh_hosts: None,
peers: BTreeMap::new(),
peer_timeout: Duration::from_secs(10),
aliases: BTreeMap::new(),
policy: Policy::default(),
catalog: BTreeMap::new(),
embed_url: None,
api_keys: ApiKeys::default(),
publish_domains: PublishDomains::Own,
log_dir: None,
series: Vec::new(),
}
}
pub fn with_base_url(mut self, url: impl Into<String>) -> Config {
self.base_url = Some(url.into());
self
}
pub fn from_env() -> Result<Config, ConfigError> {
Config::from_vars(|k| std::env::var(k).ok())
}
pub fn from_vars(get: impl Fn(&str) -> Option<String>) -> Result<Config, ConfigError> {
let host = get("DATUMESH_HOST").ok_or_else(|| ConfigError::new("DATUMESH_HOST", "not set"))?;
let mut c = Config::new(host);
if let Some(v) = get("DATUMESH_BASE_URL").filter(|v| !v.is_empty()) {
c.base_url = Some(v);
}
if let Some(v) = get("DATUMESH_NAMESPACES") {
c.namespaces = v.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
}
let ms = |k: &str, d: Duration| -> Result<Duration, ConfigError> {
match get(k) {
Some(v) => v.parse::<u64>().map(Duration::from_millis).map_err(|e| ConfigError::new(k, e.to_string())),
None => Ok(d),
}
};
let secs = |k: &str, d: Duration| -> Result<Duration, ConfigError> {
match get(k) {
Some(v) => v.parse::<u64>().map(Duration::from_secs).map_err(|e| ConfigError::new(k, e.to_string())),
None => Ok(d),
}
};
let num = |k: &str, d: usize| -> Result<usize, ConfigError> {
match get(k) {
Some(v) => v.parse::<usize>().map_err(|e| ConfigError::new(k, e.to_string())),
None => Ok(d),
}
};
let flag = |k: &str, d: bool| -> Result<bool, ConfigError> {
match get(k).as_deref() {
None => Ok(d),
Some("1" | "true" | "yes") => Ok(true),
Some("0" | "false" | "no") => Ok(false),
Some(v) => Err(ConfigError::new(k, format!("not a boolean: {v}"))),
}
};
c.budget = ms("DATUMESH_BUDGET_MS", c.budget)?;
c.pool_size = num("DATUMESH_POOL_SIZE", c.pool_size)?;
c.batch_max = num("DATUMESH_BATCH_MAX", c.batch_max)?;
c.test_cells = flag("DATUMESH_TEST_CELLS", c.test_cells)?;
c.slice_page = num("DATUMESH_SLICE_PAGE", c.slice_page)?;
c.slice_max = num("DATUMESH_SLICE_MAX", c.slice_max)?;
c.explain_depth = num("DATUMESH_EXPLAIN_DEPTH", c.explain_depth)?;
c.max_jobs = num("DATUMESH_MAX_JOBS", c.max_jobs)?;
c.trust_proxy = flag("DATUMESH_TRUST_PROXY", c.trust_proxy)?;
c.cross_mesh = flag("DATUMESH_CROSS_MESH", c.cross_mesh)?;
c.rate.enabled = flag("DATUMESH_RATE_LIMITS", c.rate.enabled)?;
c.rate.per_ip = num("DATUMESH_RATE_PER_IP", c.rate.per_ip as usize)? as u32;
c.rate.ip_burst = num("DATUMESH_RATE_IP_BURST", c.rate.ip_burst as usize)? as u32;
c.rate.per_key = num("DATUMESH_RATE_PER_KEY", c.rate.per_key as usize)? as u32;
c.rate.verify_per_ip = num("DATUMESH_RATE_VERIFY_PER_IP", c.rate.verify_per_ip as usize)? as u32;
c.rate.series_per_ip = num("DATUMESH_RATE_SERIES_PER_IP", c.rate.series_per_ip as usize)? as u32;
c.rate.calc_per_ip = num("DATUMESH_RATE_CALC_PER_IP", c.rate.calc_per_ip as usize)? as u32;
c.ttl.key_doc = secs("DATUMESH_TTL_KEY_DOC", c.ttl.key_doc)?;
c.ttl.mesh_doc = secs("DATUMESH_TTL_MESH_DOC", c.ttl.mesh_doc)?;
c.ttl.binding = secs("DATUMESH_TTL_BINDING", c.ttl.binding)?;
c.ttl.current = secs("DATUMESH_TTL_CURRENT", c.ttl.current)?;
c.ttl.latest_max_age = secs("DATUMESH_LATEST_MAX_AGE", c.ttl.latest_max_age)?;
c.ttl.job = secs("DATUMESH_JOB_TTL", c.ttl.job)?;
c.peer_timeout = ms("DATUMESH_PEER_TIMEOUT_MS", c.peer_timeout)?;
if let Some(v) = get("DATUMESH_CROSS_MESH_HOSTS") {
c.cross_mesh_hosts = Some(v.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect());
}
if let Some(v) = get("DATUMESH_PEERS") {
for pair in v.split(',').map(str::trim).filter(|s| !s.is_empty()) {
let (h, o) =
pair.split_once('=').ok_or_else(|| ConfigError::new("DATUMESH_PEERS", "host=origin pairs"))?;
c.peers.insert(h.trim().to_string(), o.trim().trim_end_matches('/').to_string());
}
}
if let Some(p) = get("DATUMESH_ALIASES_FILE") {
c.aliases = string_map(&read_file(&p)?, "DATUMESH_ALIASES_FILE")?;
}
if let Some(p) = get("DATUMESH_CATALOG_FILE") {
c.catalog = string_map(&read_file(&p)?, "DATUMESH_CATALOG_FILE")?;
}
if let Some(v) = get("DATUMESH_EMBED_URL").filter(|v| !v.trim().is_empty()) {
c.embed_url = Some(v.trim().trim_end_matches('/').to_string());
}
if let Some(p) = get("DATUMESH_POLICY_FILE") {
c.policy = Policy::from_json(&read_file(&p)?)?;
}
if let Some(p) = get("DATUMESH_API_KEYS_FILE") {
c.api_keys = ApiKeys::parse(&read_secret_file(Path::new(&p))?)?;
}
c.publish_domains = PublishDomains::parse(&get("DATUMESH_PUBLISH_DOMAINS").unwrap_or_default())?;
if let Some(p) = get("DATUMESH_SERIES_FILE").filter(|p| !p.trim().is_empty()) {
c.series = crate::series::parse(&read_file(&p)?)?;
}
if let Some(d) = get("DATUMESH_LOG_DIR").filter(|d| !d.trim().is_empty()) {
c.log_dir = Some(std::path::PathBuf::from(d));
}
c.check()?;
Ok(c)
}
pub fn check(&self) -> Result<(), ConfigError> {
lexical::check_dns_name(&self.host, "$").map_err(|e| ConfigError::new("host", e.to_string()))?;
if let Some(u) = &self.base_url {
check_base_url(&self.host, u).map_err(|e| ConfigError::new("base URL", format!("{u}: {e}")))?;
}
for h in &self.namespaces {
lexical::check_dns_name(h, "$").map_err(|e| ConfigError::new("namespaces", e.to_string()))?;
}
for (h, o) in &self.peers {
lexical::check_dns_name(h, "$").map_err(|e| ConfigError::new("peers", e.to_string()))?;
if !(o.starts_with("http://") || o.starts_with("https://")) {
return Err(ConfigError::new("peers", format!("{o}: not an http(s) origin")));
}
}
for (a, n) in &self.aliases {
lexical::check_function_name(n, "$").map_err(|e| ConfigError::new("aliases", format!("{a}: {e}")))?;
}
if self.pool_size == 0 || self.batch_max == 0 || self.slice_page == 0 || self.slice_max == 0 {
return Err(ConfigError::new("sizes", "the pool, batch and slice bounds must be positive"));
}
if self.ttl.job < Duration::from_secs(300) {
return Err(ConfigError::new("job TTL", "at least 300 s (SPEC 5.5)"));
}
if self.budget.is_zero() {
return Err(ConfigError::new("budget", "must be positive"));
}
Ok(())
}
pub fn owns(&self, host: &str) -> bool {
self.namespaces.iter().any(|h| h == host)
}
pub fn origin(&self) -> String {
format!("https://{}", self.host)
}
pub fn mesh_document_base(&self) -> String {
self.base_url.clone().unwrap_or_else(|| self.origin())
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BaseUrlError {
Scheme,
HttpRootedHost,
HttpNotLoopback,
Host,
Port,
Path,
TrailingSlash,
Query,
Fragment,
}
impl fmt::Display for BaseUrlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
BaseUrlError::Scheme => "not an https:// or http:// URL",
BaseUrlError::HttpRootedHost => "http:// is for an unrooted host (under .invalid) only",
BaseUrlError::HttpNotLoopback => "http:// is for 127.0.0.1, [::1] and localhost only",
BaseUrlError::Host => "not a host name, IPv4 address or bracketed IPv6 address",
BaseUrlError::Port => "not a port from 1 to 65535",
BaseUrlError::Path => "a path (the base URL is a scheme, a host and a port)",
BaseUrlError::TrailingSlash => "a trailing slash",
BaseUrlError::Query => "a query",
BaseUrlError::Fragment => "a fragment",
})
}
}
impl std::error::Error for BaseUrlError {}
pub fn check_base_url(host: &str, url: &str) -> Result<(), BaseUrlError> {
let mesh_host = host;
let (http, rest) = if let Some(r) = url.strip_prefix("https://") {
(false, r)
} else if let Some(r) = url.strip_prefix("http://") {
(true, r)
} else {
return Err(BaseUrlError::Scheme);
};
if rest.contains('#') {
return Err(BaseUrlError::Fragment);
}
if rest.contains('?') {
return Err(BaseUrlError::Query);
}
let (authority, path) = rest.split_at(rest.find('/').unwrap_or(rest.len()));
match path {
"" => {}
"/" => return Err(BaseUrlError::TrailingSlash),
_ => return Err(BaseUrlError::Path),
}
if authority.contains('@') {
return Err(BaseUrlError::Host);
}
let (host, port) = if let Some(r) = authority.strip_prefix('[') {
let (inside, after) = r.split_once(']').ok_or(BaseUrlError::Host)?;
inside.parse::<std::net::Ipv6Addr>().map_err(|_| BaseUrlError::Host)?;
let port = match after {
"" => None,
p => Some(p.strip_prefix(':').ok_or(BaseUrlError::Host)?),
};
(&authority[..inside.len() + 2], port)
} else {
let (host, port) = match authority.split_once(':') {
Some((h, p)) => (h, Some(p)),
None => (authority, None),
};
if host.parse::<std::net::Ipv4Addr>().is_err() && !is_dns_name(host) {
return Err(BaseUrlError::Host);
}
(host, port)
};
if let Some(p) = port {
let ok = !p.is_empty() && p.len() <= 5 && p.bytes().all(|b| b.is_ascii_digit());
if !ok || !(1..=65535).contains(&p.parse::<u32>().unwrap_or(0)) {
return Err(BaseUrlError::Port);
}
}
if http && !is_unrooted(mesh_host) {
return Err(BaseUrlError::HttpRootedHost);
}
if http && !matches!(host, "127.0.0.1" | "[::1]" | "localhost") {
return Err(BaseUrlError::HttpNotLoopback);
}
Ok(())
}
pub fn is_unrooted(host: &str) -> bool {
host.rsplit('.').next() == Some("invalid")
}
fn is_dns_name(h: &str) -> bool {
!h.is_empty()
&& h.len() <= 253
&& h.split('.').all(|l| {
!l.is_empty()
&& l.len() <= 63
&& !l.starts_with('-')
&& !l.ends_with('-')
&& l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
})
}
fn string_map(bytes: &[u8], what: &str) -> Result<BTreeMap<String, String>, ConfigError> {
let v = json::parse(bytes).map_err(|e| ConfigError::new(what, e.to_string()))?;
let mut out = BTreeMap::new();
for (k, x) in v.as_object().ok_or_else(|| ConfigError::new(what, "not an object"))? {
let s = x.as_str().ok_or_else(|| ConfigError::new(what, format!("{k}: not a string")))?;
out.insert(k.clone(), s.to_string());
}
Ok(out)
}
fn read_file(path: &str) -> Result<Vec<u8>, ConfigError> {
std::fs::read(path).map_err(|e| ConfigError::new(path, e.to_string()))
}
pub fn read_secret_file(path: &Path) -> Result<Vec<u8>, ConfigError> {
let what = path.display().to_string();
let meta = std::fs::metadata(path).map_err(|e| ConfigError::new(&what, e.to_string()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = meta.permissions().mode() & 0o777;
if mode & 0o077 != 0 {
return Err(ConfigError::new(&what, format!("mode {mode:o}: a secret file must be 0600")));
}
}
#[cfg(not(unix))]
let _ = meta;
std::fs::read(path).map_err(|e| ConfigError::new(&what, e.to_string()))
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ConfigError {
what: String,
detail: String,
}
impl ConfigError {
pub(crate) fn new(what: &str, detail: impl Into<String>) -> ConfigError {
ConfigError { what: what.to_string(), detail: detail.into() }
}
}
impl fmt::Display for ConfigError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "configuration: {}: {}", self.what, self.detail)
}
}
impl std::error::Error for ConfigError {}