pub use crate::filter::{FilterCondition, FilterOp};
use crate::{Error, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::{BTreeSet, HashMap};
use std::path::{Path, PathBuf};
use std::{fs, io};
fn from_json_value<T, E>(value: Value) -> std::result::Result<T, E>
where
T: serde::de::DeserializeOwned,
E: serde::de::Error,
{
T::deserialize(value).map_err(serde::de::Error::custom)
}
fn shape_error<E: serde::de::Error>(directive: &str, expected: &str, got: &Value) -> E {
serde::de::Error::custom(format!(
"{} must be {}, got {}",
directive,
expected,
crate::value_kind(got)
))
}
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum EmitTarget {
List,
Ids,
}
#[derive(Debug, Clone, PartialEq)]
pub enum AggregateSpec {
Paths(Vec<String>),
Config(AggregateConfig),
}
impl<'de> Deserialize<'de> for AggregateSpec {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
match Value::deserialize(deserializer)? {
value @ Value::Array(_) => from_json_value(value).map(AggregateSpec::Paths),
value @ Value::Object(_) => from_json_value(value).map(AggregateSpec::Config),
other => Err(shape_error(
"$aggregate",
"a list of source paths or an object",
&other,
)),
}
}
}
#[derive(Deserialize, Debug, Clone, PartialEq)]
pub struct AggregateConfig {
#[serde(default)]
pub mode: AggregateMode,
pub sources: Vec<AggregateSource>,
}
#[derive(Deserialize, Debug, Clone, PartialEq, Eq, Default)]
#[serde(rename_all = "lowercase")]
pub enum AggregateMode {
#[default]
Flat,
Keyed,
}
#[derive(Debug, Clone, PartialEq)]
pub enum AggregateSource {
Path(String),
Mapping(AggregateSourceMapping),
}
impl<'de> Deserialize<'de> for AggregateSource {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
match Value::deserialize(deserializer)? {
Value::String(path) => Ok(AggregateSource::Path(path)),
value @ Value::Object(_) => from_json_value(value).map(AggregateSource::Mapping),
other => Err(shape_error(
"$aggregate source entry",
"a source path or an object",
&other,
)),
}
}
}
#[derive(Deserialize, Debug, Clone, PartialEq)]
pub struct AggregateSourceMapping {
pub from: String,
#[serde(rename = "as")]
pub as_key: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AggregateEntry {
pub from: String,
pub key: Option<String>,
}
impl AggregateSpec {
pub fn mode(&self) -> AggregateMode {
match self {
AggregateSpec::Paths(_) => AggregateMode::Flat,
AggregateSpec::Config(cfg) => cfg.mode.clone(),
}
}
pub fn entries(&self) -> Vec<AggregateEntry> {
match self {
AggregateSpec::Paths(paths) => paths
.iter()
.map(|p| AggregateEntry {
from: p.clone(),
key: None,
})
.collect(),
AggregateSpec::Config(cfg) => cfg
.sources
.iter()
.map(|s| match s {
AggregateSource::Path(p) => AggregateEntry {
from: p.clone(),
key: None,
},
AggregateSource::Mapping(m) => AggregateEntry {
from: m.from.clone(),
key: m.as_key.clone(),
},
})
.collect(),
}
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ApiNode {
pub filter: Option<Vec<FilterCondition>>,
pub aggregate: Option<AggregateSpec>,
pub pick: Option<Vec<String>>,
pub omit: Option<Vec<String>>,
pub skip: Option<bool>,
pub emit: Option<Vec<EmitTarget>>,
pub values: Option<Vec<Value>>,
pub derive: Option<DeriveSource>,
pub sub_paths: HashMap<String, ApiNode>,
}
const NODE_DIRECTIVES: [&str; 8] = [
"$filter",
"$aggregate",
"$pick",
"$omit",
"$skip",
"$emit",
"$values",
"$derive",
];
impl<'de> Deserialize<'de> for ApiNode {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct ApiNodeVisitor;
impl<'de> serde::de::Visitor<'de> for ApiNodeVisitor {
type Value = ApiNode;
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("an object of directives and sub-paths")
}
fn visit_map<M>(self, mut map: M) -> std::result::Result<ApiNode, M::Error>
where
M: serde::de::MapAccess<'de>,
{
use serde::de::Error;
let mut node = ApiNode::default();
macro_rules! once {
($field:expr, $key:expr, $map:expr) => {{
if $field.is_some() {
return Err(Error::custom(format!("duplicate {}", $key)));
}
$field = Some($map.next_value()?);
}};
}
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"$filter" => once!(node.filter, key, map),
"$aggregate" => once!(node.aggregate, key, map),
"$pick" => once!(node.pick, key, map),
"$omit" => once!(node.omit, key, map),
"$skip" => once!(node.skip, key, map),
"$emit" => once!(node.emit, key, map),
"$values" => once!(node.values, key, map),
"$derive" => once!(node.derive, key, map),
unknown
if unknown.starts_with('$')
&& template_var_from_key(unknown).is_none() =>
{
return Err(Error::custom(format!(
"unknown directive {}, expected one of {}",
unknown,
NODE_DIRECTIVES.join(", ")
)));
}
_ => {
let child = map.next_value()?;
if node.sub_paths.insert(key.clone(), child).is_some() {
return Err(Error::custom(format!("duplicate sub-path '{}'", key)));
}
}
}
}
Ok(node)
}
}
deserializer.deserialize_map(ApiNodeVisitor)
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum DeriveSource {
Field(String),
Config(DeriveConfig),
}
impl<'de> Deserialize<'de> for DeriveSource {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
match Value::deserialize(deserializer)? {
Value::String(field) => Ok(DeriveSource::Field(field)),
value @ Value::Object(_) => from_json_value(value).map(DeriveSource::Config),
other => Err(shape_error("$derive", "a field name or an object", &other)),
}
}
}
#[derive(Deserialize, Debug, Clone, PartialEq)]
pub struct DeriveConfig {
pub field: String,
pub pattern: Option<String>,
#[serde(default, rename = "type")]
pub value_type: Option<DeriveType>,
#[serde(default)]
pub exclude: Option<Vec<Value>>,
}
#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum DeriveType {
String,
Int,
}
#[derive(Debug, Clone, PartialEq)]
pub enum StaticSpec {
Include(Vec<String>),
Detailed(StaticConfig),
}
impl<'de> Deserialize<'de> for StaticSpec {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
match Value::deserialize(deserializer)? {
value @ Value::Array(_) => from_json_value(value).map(StaticSpec::Include),
value @ Value::Object(_) => from_json_value(value).map(StaticSpec::Detailed),
other => Err(shape_error(
"$static",
"a list of globs or an object",
&other,
)),
}
}
}
#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
pub struct StaticConfig {
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
}
impl StaticSpec {
pub fn include(&self) -> &[String] {
match self {
StaticSpec::Include(patterns) => patterns,
StaticSpec::Detailed(cfg) => &cfg.include,
}
}
pub fn exclude(&self) -> &[String] {
match self {
StaticSpec::Include(_) => &[],
StaticSpec::Detailed(cfg) => &cfg.exclude,
}
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default, clap::ValueEnum)]
#[serde(rename_all = "lowercase")]
pub enum Layout {
#[default]
Index,
File,
Extension,
}
#[derive(Deserialize)]
pub struct SerializerConfig {
pub serializer: String,
#[serde(default)]
pub layout: Layout,
pub dest: PathBuf,
#[serde(default)]
pub bundle: bool,
#[serde(default)]
pub minify: bool,
#[serde(default)]
pub overwrite: bool,
}
#[derive(Deserialize)]
pub struct Config {
#[serde(default, rename = "$config")]
pub serializers: Vec<SerializerConfig>,
#[serde(default, rename = "$static")]
pub static_files: Option<StaticSpec>,
#[serde(skip)]
pub copy_static_all: bool,
#[serde(flatten)]
pub api: HashMap<String, ApiNode>,
}
impl Default for Config {
fn default() -> Self {
Self {
serializers: vec![SerializerConfig {
serializer: "json".into(),
layout: Layout::Index,
dest: "dist".into(),
bundle: false,
minify: false,
overwrite: false,
}],
static_files: None,
copy_static_all: false,
api: HashMap::new(),
}
}
}
impl Config {
pub fn new<P: AsRef<Path>>(serializer: String, layout: Layout, dest: P) -> Self {
let dest = dest.as_ref().to_path_buf();
Self {
serializers: vec![SerializerConfig {
serializer,
layout,
dest,
bundle: false,
minify: false,
overwrite: false,
}],
static_files: None,
copy_static_all: false,
api: HashMap::new(),
}
}
pub fn load_from_str<S: AsRef<str>>(s: S) -> Result<Self> {
let content = s.as_ref();
let config: Self = serde_json::from_str(content).map_err(Error::SerdeJson)?;
config.validate()?;
Ok(config)
}
pub fn load_from_reader(reader: &mut impl std::io::Read) -> Result<Self> {
let mut reader = std::io::BufReader::new(reader);
let content = io::read_to_string(&mut reader)?;
Self::load_from_str(content)
}
pub fn load_from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
let content = fs::read_to_string(path).map_err(Error::Io)?;
Self::load_from_str(content)
}
}
impl Config {
fn validate(&self) -> Result<()> {
if let Some(spec) = self.static_files.as_ref() {
validate_static(spec)?;
}
let mut keys = self.api.keys().cloned().collect::<Vec<_>>();
keys.sort();
for key in keys {
if let Some(node) = self.api.get(&key) {
validate_node(&key, node)?;
}
}
Ok(())
}
}
fn validate_static(spec: &StaticSpec) -> Result<()> {
for pattern in spec.include().iter().chain(spec.exclude().iter()) {
globset::Glob::new(pattern).map_err(|e| {
Error::Config(format!("invalid $static glob pattern '{}': {}", pattern, e))
})?;
}
Ok(())
}
fn validate_node(path: &str, node: &ApiNode) -> Result<()> {
if let Some(aggregate) = node.aggregate.as_ref() {
validate_aggregate(path, aggregate)?;
}
if let Some(filters) = node.filter.as_ref() {
validate_filter(path, filters)?;
}
let mut keys = node.sub_paths.keys().cloned().collect::<Vec<_>>();
keys.sort();
for key in keys {
let child = node
.sub_paths
.get(&key)
.ok_or_else(|| Error::Config(format!("{}: missing child node {}", path, key)))?;
let child_path = format!("{}/{}", path, key);
if template_var_from_key(&key).is_some() {
if child.values.is_some() && child.derive.is_some() {
return Err(Error::Config(format!(
"{}: $values and $derive cannot be used together",
child_path
)));
}
if child.values.is_none() && child.derive.is_none() {
return Err(Error::Config(format!(
"{}: template sub-path requires $values or $derive",
child_path
)));
}
if let Some(values) = child.values.as_ref() {
if values.is_empty() {
return Err(Error::Config(format!(
"{}: $values must not be empty",
child_path
)));
}
for value in values {
if !is_scalar(value) {
return Err(Error::Config(format!(
"{}: $values entries must be scalar (string/number/bool)",
child_path
)));
}
if let Value::String(s) = value
&& s.contains('/')
{
return Err(Error::Config(format!(
"{}: $values string must not contain '/'",
child_path
)));
}
}
}
if let Some(derive) = child.derive.as_ref() {
validate_derive(&child_path, derive)?;
}
} else if child.values.is_some() || child.derive.is_some() {
return Err(Error::Config(format!(
"{}: $values/$derive are only allowed for template sub-path keys like ${{name}}",
child_path
)));
}
validate_node(&child_path, child)?;
}
Ok(())
}
fn validate_aggregate(path: &str, aggregate: &AggregateSpec) -> Result<()> {
let entries = aggregate.entries();
if entries.is_empty() {
return Err(Error::Config(format!(
"{}: $aggregate must not be empty",
path
)));
}
let mode = aggregate.mode();
let mut keyed_names = BTreeSet::new();
for entry in entries {
if entry.from.trim().is_empty() {
return Err(Error::Config(format!(
"{}: $aggregate source must not be empty",
path
)));
}
if mode == AggregateMode::Keyed {
let key = entry.key.unwrap_or(entry.from);
if key.trim().is_empty() {
return Err(Error::Config(format!(
"{}: $aggregate keyed source alias must not be empty",
path
)));
}
if !keyed_names.insert(key.clone()) {
return Err(Error::Config(format!(
"{}: duplicate keyed aggregate key '{}'",
path, key
)));
}
}
}
Ok(())
}
fn validate_filter(path: &str, filters: &[FilterCondition]) -> Result<()> {
for cond in filters {
if !matches!(cond.op, FilterOp::RegEq | FilterOp::RegNeq) {
continue;
}
let pattern = cond.value.as_str().ok_or_else(|| {
Error::Config(format!(
"{}: $filter {} on '{}' requires a string value, got {}",
path,
cond.op,
cond.field,
crate::value_kind(&cond.value)
))
})?;
crate::compile_regex(pattern).map_err(|e| {
Error::Config(format!(
"{}: invalid $filter {} pattern '{}' on '{}': {}",
path, cond.op, pattern, cond.field, e
))
})?;
}
Ok(())
}
fn template_var_from_key(key: &str) -> Option<&str> {
if key.starts_with("${") && key.ends_with('}') && key.len() > 3 {
Some(&key[2..key.len() - 1])
} else {
None
}
}
fn is_scalar(value: &Value) -> bool {
matches!(value, Value::String(_) | Value::Number(_) | Value::Bool(_))
}
fn validate_derive(path: &str, derive: &DeriveSource) -> Result<()> {
let cfg = derive.to_config();
if cfg.field.trim().is_empty() {
return Err(Error::Config(format!(
"{}: $derive.field must not be empty",
path
)));
}
if let Some(pattern) = cfg.pattern.as_ref() {
crate::compile_regex(pattern).map_err(|e| {
Error::Config(format!(
"{}: invalid $derive.pattern '{}': {}",
path, pattern, e
))
})?;
}
for value in cfg.exclude.iter().flatten() {
if value.is_array() || value.is_object() {
return Err(Error::Config(format!(
"{}: $derive.exclude takes scalars, got {}",
path,
crate::value_kind(value)
)));
}
}
Ok(())
}
impl DeriveSource {
pub fn to_config(&self) -> DeriveConfig {
match self {
DeriveSource::Field(field) => DeriveConfig {
field: field.clone(),
pattern: None,
value_type: None,
exclude: None,
},
DeriveSource::Config(c) => c.clone(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn test_parse_advanced_routing_config() {
let config_path = Path::new("testdata/tamada/_config.json");
let config =
Config::load_from_file(config_path).expect("Failed to load complex configuration");
let job_hist = config
.api
.get("job-histories")
.expect("Missing job-histories node");
let current = job_hist
.sub_paths
.get("current")
.expect("Missing current sub-path");
let filter = current.filter.as_ref().expect("Missing filter array");
assert_eq!(filter.len(), 1);
assert_eq!(filter[0].field, "to");
assert_eq!(filter[0].op, FilterOp::Eq);
assert_eq!(filter[0].value, Value::String("Present".to_string()));
let activities = config
.api
.get("activities")
.expect("Missing activities node");
assert_eq!(activities.filter, None);
assert_eq!(activities.emit, None);
let by_year = activities
.sub_paths
.get("${year}")
.expect("Missing ${year} sub-path");
let derive_config = by_year
.derive
.as_ref()
.expect("Missing $derive")
.to_config();
assert_eq!(derive_config.field, "from");
assert_eq!(derive_config.pattern, Some("^(\\d{4}).*".to_string()));
let profile = config.api.get("profile").expect("Missing profile node");
assert_eq!(profile.emit, None);
let agg = profile.aggregate.as_ref().expect("Missing aggregate array");
assert_eq!(agg.mode(), AggregateMode::Keyed);
let entries = agg.entries();
assert_eq!(entries[0].from, "job-histories");
assert_eq!(entries[1].from, "activities");
assert_eq!(entries[2].from, "degrees");
assert_eq!(entries[3].from, "skills");
let secret = config.api.get("secret").expect("Missing secret node");
assert_eq!(secret.emit, Some(vec![]));
}
#[test]
fn test_parse_template_derive_config() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"activities": {{
"${{year}}": {{
"$derive": {{"field":"from", "pattern":"^(\\d{{4}})"}}
}}
}}
}}"#
)
.unwrap();
let config =
Config::load_from_file(tmp.path()).expect("Failed to load derive configuration");
let activities = config
.api
.get("activities")
.expect("Missing activities node");
let by_year = activities
.sub_paths
.get("${year}")
.expect("Missing template node");
let derive = by_year.derive.as_ref().expect("Missing derive").to_config();
assert_eq!(derive.field, "from");
assert_eq!(derive.pattern, Some("^(\\d{4})".to_string()));
}
#[test]
fn test_parse_derive_value_type() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"papers": {{
"${{year}}": {{
"$derive": {{"field":"published", "pattern":"^(\\d{{4}})", "type":"int"}}
}},
"${{tag}}": {{
"$derive": {{"field":"tag"}}
}}
}}
}}"#
)
.unwrap();
let config = Config::load_from_file(tmp.path()).expect("Failed to load derive type config");
let papers = config.api.get("papers").expect("Missing papers node");
let by_year = papers
.sub_paths
.get("${year}")
.expect("Missing ${year} sub-path");
let derive = by_year.derive.as_ref().expect("Missing derive").to_config();
assert_eq!(derive.field, "published");
assert_eq!(derive.value_type, Some(DeriveType::Int));
let by_tag = papers
.sub_paths
.get("${tag}")
.expect("Missing ${tag} sub-path");
let derive = by_tag.derive.as_ref().expect("Missing derive").to_config();
assert_eq!(derive.value_type, None);
}
#[test]
fn test_unknown_derive_value_type_is_rejected() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"papers": {{
"${{year}}": {{
"$derive": {{"field":"published", "type":"integer"}}
}}
}}
}}"#
)
.unwrap();
assert!(Config::load_from_file(tmp.path()).is_err());
}
#[test]
fn test_composite_derive_exclude_entry_is_rejected() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"papers": {{
"${{year}}": {{
"$derive": {{"field":"published", "exclude":[[2024]]}}
}}
}}
}}"#
)
.unwrap();
let err = match Config::load_from_file(tmp.path()) {
Ok(_) => panic!("a composite exclude entry should be rejected"),
Err(e) => e.to_string(),
};
assert!(
err.contains("$derive.exclude") && err.contains("array"),
"error should name the directive and the offending kind, got: {}",
err
);
}
#[test]
fn test_scalar_derive_exclude_entries_are_accepted() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"papers": {{
"${{year}}": {{
"$derive": {{"field":"published", "exclude":[0, "draft", true]}}
}}
}}
}}"#
)
.unwrap();
let config =
Config::load_from_file(tmp.path()).expect("scalar exclude entries should load");
let derive = config
.api
.get("papers")
.and_then(|n| n.sub_paths.get("${year}"))
.and_then(|n| n.derive.as_ref())
.expect("Missing $derive")
.to_config();
assert_eq!(
derive.exclude,
Some(vec![
serde_json::json!(0),
serde_json::json!("draft"),
serde_json::json!(true)
])
);
}
#[test]
fn test_unknown_directive_is_rejected() {
for body in [
r#"{"$fliter": {}}"#,
r#"{"$fliter": true}"#,
r#"{"$emitt": []}"#,
] {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"users": {}
}}"#,
body
)
.unwrap();
let err = match Config::load_from_file(tmp.path()) {
Ok(_) => panic!("{} should be rejected", body),
Err(e) => e.to_string(),
};
assert!(
err.contains("unknown directive"),
"{} should be reported as an unknown directive, got: {}",
body,
err
);
}
}
#[test]
fn test_unknown_directive_error_names_the_key_and_the_alternatives() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"users": {{"$fliter": {{}}}}
}}"#
)
.unwrap();
let err = match Config::load_from_file(tmp.path()) {
Ok(_) => panic!("a misspelled directive should be rejected"),
Err(e) => e.to_string(),
};
assert!(err.contains("$fliter"), "should name the key, got: {}", err);
assert!(
err.contains("$filter") && err.contains("$derive"),
"should list the directives, got: {}",
err
);
}
#[test]
fn test_template_sub_path_key_is_not_treated_as_a_directive() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"papers": {{"${{year}}": {{"$derive": "year"}}}}
}}"#
)
.unwrap();
let config = Config::load_from_file(tmp.path()).expect("template keys must still load");
assert!(
config
.api
.get("papers")
.expect("Missing papers node")
.sub_paths
.contains_key("${year}")
);
}
#[test]
fn test_duplicate_directive_is_rejected() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"users": {{"$pick": ["id"], "$pick": ["name"]}}
}}"#
)
.unwrap();
let err = match Config::load_from_file(tmp.path()) {
Ok(_) => panic!("a repeated directive should be rejected"),
Err(e) => e.to_string(),
};
assert!(
err.contains("duplicate") && err.contains("$pick"),
"should name the repeated directive, got: {}",
err
);
}
#[test]
fn test_non_template_derive_is_rejected() {
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
r#"{{
"$config": [{{"serializer":"json","layout":"index","dest":"dist"}}],
"activities": {{
"by-year": {{
"$derive": "from"
}}
}}
}}"#
)
.unwrap();
let err = match Config::load_from_file(tmp.path()) {
Ok(_) => panic!("config should be rejected"),
Err(e) => e,
};
assert!(format!("{}", err).contains("$values/$derive are only allowed"));
}
}