use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use super::{
FieldProvenance, KnownFields, OUTPUT_DEPENDS_ON_INPUT_NAME, ProcessorDefaults, ProvenanceMap,
SCAN_CONFIG_FIELDS, STANDARD_EXTRA_FIELDS, ScanDefaultsData, StandardConfig, provenance,
};
use crate::registries::{self as registry, ProcessorPlugin};
#[derive(Debug, Clone)]
pub struct ProcessorInstance {
pub instance_name: String,
pub pname: String,
pub config_toml: toml::Value,
pub provenance: ProvenanceMap,
}
pub fn find_registry_entry(pname: &str) -> Option<&'static ProcessorPlugin> {
registry::find_plugin(pname)
}
pub fn registry_entries() -> impl Iterator<Item = &'static ProcessorPlugin> {
registry::all_plugins()
}
pub fn is_builtin_type(name: &str) -> bool {
find_registry_entry(name).is_some()
}
pub fn seed_user_provenance(value: &toml::Value) -> ProvenanceMap {
let mut map = ProvenanceMap::new();
if let Some(table) = value.as_table() {
for key in table.keys() {
map.insert(key.clone(), FieldProvenance::UserToml { line: 0 });
}
}
map
}
pub fn resolve_instance_defaults(
type_name: &str,
value: &mut toml::Value,
provenance: &mut ProvenanceMap,
) {
if find_registry_entry(type_name).is_some() {
registry::apply_all_defaults(type_name, value, provenance);
}
}
impl ProcessorConfig {
pub(crate) fn src_dirs(&self) -> Vec<String> {
let mut dirs: Vec<String> = self
.instances
.iter()
.flat_map(|inst| {
inst.config_toml
.get("src_dirs")
.and_then(|v| v.as_array())
.into_iter()
.flat_map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(std::string::ToString::to_string))
})
.filter(|d| !d.is_empty())
})
.collect();
dirs.sort();
dirs.dedup();
dirs
}
pub(crate) fn known_fields_for(type_name: &str) -> Option<Vec<&'static str>> {
let e = find_registry_entry(type_name)?;
let mut fields: Vec<&'static str> = StandardConfig::known_fields()
.iter()
.copied()
.filter(|f| !e.omit_standard_fields.contains(f))
.collect();
for spec in e.fields {
if !fields.contains(&spec.name) {
fields.push(spec.name);
}
}
Some(fields)
}
pub(crate) fn checksum_fields_for(type_name: &str) -> Option<Vec<&'static str>> {
let e = find_registry_entry(type_name)?;
let shadowed = |f: &&'static str| e.fields.iter().any(|s| s.name == *f);
let mut fields: Vec<&'static str> = StandardConfig::checksum_fields()
.iter()
.copied()
.filter(|f| !e.omit_standard_fields.contains(f))
.filter(|f| !shadowed(f))
.collect();
for spec in e.fields {
if spec.affects_output {
fields.push(spec.name);
}
}
Some(fields)
}
pub(crate) fn must_fields_for(type_name: &str) -> Option<Vec<&'static str>> {
let e = find_registry_entry(type_name)?;
Some(
e.fields
.iter()
.filter(|s| s.required)
.map(|s| s.name)
.collect(),
)
}
pub(crate) fn field_descriptions_for(
type_name: &str,
) -> Option<Vec<(&'static str, &'static str)>> {
let e = find_registry_entry(type_name)?;
let shadowed = |f: &str| e.fields.iter().any(|s| s.name == f);
let mut descs: Vec<(&'static str, &'static str)> = StandardConfig::field_descriptions()
.iter()
.copied()
.filter(|(f, _)| !e.omit_standard_fields.contains(f) && !shadowed(f))
.collect();
for spec in e.fields {
descs.push((spec.name, spec.doc));
}
Some(descs)
}
pub(crate) fn defconfig_json(type_name: &str) -> Option<String> {
let entry = find_registry_entry(type_name)?;
(entry.defconfig_json)(&entry.pname())
}
}
pub fn scan_defaults_for(type_name: &str) -> Option<ScanDefaultsData> {
find_registry_entry(type_name)?.scan_defaults
}
pub fn processor_defaults_for(type_name: &str) -> Option<ProcessorDefaults> {
find_registry_entry(type_name)?.defaults
}
pub fn apply_processor_defaults(
type_name: &str,
value: &mut toml::Value,
provenance: &mut ProvenanceMap,
) {
let Some(defaults) = processor_defaults_for(type_name) else {
return;
};
let Some(table) = value.as_table_mut() else {
return;
};
set_string_default(
table,
"command",
defaults.command,
provenance,
FieldProvenance::ProcessorDefault,
);
set_string_default(
table,
"output_dir",
defaults.output_dir,
provenance,
FieldProvenance::ProcessorDefault,
);
set_string_array_default(
table,
"dep_auto",
defaults.dep_auto,
provenance,
FieldProvenance::ProcessorDefault,
);
set_string_array_default(
table,
"formats",
defaults.formats,
provenance,
FieldProvenance::ProcessorDefault,
);
set_string_array_default(
table,
"args",
defaults.args,
provenance,
FieldProvenance::ProcessorDefault,
);
if let Some(batch) = defaults.batch
&& !table.contains_key("batch")
{
table.insert("batch".into(), toml::Value::Boolean(batch));
provenance::record_if_absent(provenance, "batch", FieldProvenance::ProcessorDefault);
}
if defaults.output_depends_on_input_name && !table.contains_key(OUTPUT_DEPENDS_ON_INPUT_NAME) {
table.insert(
OUTPUT_DEPENDS_ON_INPUT_NAME.into(),
toml::Value::Boolean(true),
);
provenance::record_if_absent(
provenance,
OUTPUT_DEPENDS_ON_INPUT_NAME,
FieldProvenance::ProcessorDefault,
);
}
}
fn set_string_default(
table: &mut toml::map::Map<String, toml::Value>,
key: &str,
val: &str,
provenance: &mut ProvenanceMap,
source: FieldProvenance,
) {
if !val.is_empty() && !table.contains_key(key) {
table.insert(key.into(), toml::Value::String(val.into()));
provenance::record_if_absent(provenance, key, source);
}
}
fn set_string_array_default(
table: &mut toml::map::Map<String, toml::Value>,
key: &str,
vals: &[&str],
provenance: &mut ProvenanceMap,
source: FieldProvenance,
) {
if !vals.is_empty() && !table.contains_key(key) {
let arr: Vec<toml::Value> = vals
.iter()
.map(|s| toml::Value::String(s.to_string()))
.collect();
table.insert(key.into(), toml::Value::Array(arr));
provenance::record_if_absent(provenance, key, source);
}
}
fn set_empty_array_default(
table: &mut toml::map::Map<String, toml::Value>,
key: &str,
provenance: &mut ProvenanceMap,
source: FieldProvenance,
) {
if !table.contains_key(key) {
table.insert(key.into(), toml::Value::Array(Vec::new()));
provenance::record_if_absent(provenance, key, source);
}
}
fn set_maybe_empty_array_default(
table: &mut toml::map::Map<String, toml::Value>,
key: &str,
vals: &[&str],
provenance: &mut ProvenanceMap,
source: FieldProvenance,
) {
if !table.contains_key(key) {
let arr: Vec<toml::Value> = vals
.iter()
.map(|s| toml::Value::String(s.to_string()))
.collect();
table.insert(key.into(), toml::Value::Array(arr));
provenance::record_if_absent(provenance, key, source);
}
}
pub fn apply_scan_defaults(
type_name: &str,
value: &mut toml::Value,
provenance: &mut ProvenanceMap,
) {
let Some(defaults) = scan_defaults_for(type_name) else {
return;
};
let Some(table) = value.as_table_mut() else {
return;
};
set_maybe_empty_array_default(
table,
"src_dirs",
defaults.src_dirs,
provenance,
FieldProvenance::ScanDefault,
);
set_maybe_empty_array_default(
table,
"src_extensions",
defaults.src_extensions,
provenance,
FieldProvenance::ScanDefault,
);
set_maybe_empty_array_default(
table,
"src_exclude_dirs",
defaults.src_exclude_dirs,
provenance,
FieldProvenance::ScanDefault,
);
set_empty_array_default(
table,
"src_exclude_files",
provenance,
FieldProvenance::ScanDefault,
);
set_empty_array_default(
table,
"src_exclude_paths",
provenance,
FieldProvenance::ScanDefault,
);
set_empty_array_default(table, "src_files", provenance, FieldProvenance::ScanDefault);
}
#[derive(Debug, Default)]
pub struct ProcessorConfig {
pub instances: Vec<ProcessorInstance>,
}
impl Serialize for ProcessorConfig {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
let mut root = toml::map::Map::new();
for inst in &self.instances {
let Some(parsed) = registry::parse_name(&inst.instance_name) else {
continue;
};
let type_table = root
.entry(parsed.processor_type.as_str().to_string())
.or_insert_with(|| toml::Value::Table(toml::map::Map::new()));
let Some(type_table) = type_table.as_table_mut() else {
continue;
};
match parsed.instance {
None => {
type_table.insert(parsed.short.to_string(), inst.config_toml.clone());
}
Some(instance) => {
let name_table = type_table
.entry(parsed.short.to_string())
.or_insert_with(|| toml::Value::Table(toml::map::Map::new()));
if let Some(name_table) = name_table.as_table_mut() {
name_table.insert(instance.to_string(), inst.config_toml.clone());
}
}
}
}
toml::Value::Table(root).serialize(serializer)
}
}
impl<'de> Deserialize<'de> for ProcessorConfig {
fn deserialize<D: serde::Deserializer<'de>>(
deserializer: D,
) -> std::result::Result<Self, D::Error> {
let table = toml::Value::deserialize(deserializer)?;
Ok(Self::from_toml(&table))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SectionShape {
SingleInstance,
MultiInstance,
Ambiguous { colliding: Vec<String> },
}
impl ProcessorConfig {
pub(crate) fn from_toml(value: &toml::Value) -> Self {
let Some(table) = value.as_table() else {
return Self::default();
};
let mut instances = Vec::new();
for (type_key, type_val) in table {
let Some(type_table) = type_val.as_table() else {
continue;
};
if crate::processor::ProcessorType::parse(type_key).is_none() {
continue;
}
for (short, val) in type_table {
let Some(sub_table) = val.as_table() else {
continue;
};
let pname = format!("{}{type_key}.{short}", registry::NAME_PREFIX);
if !is_builtin_type(&pname) {
continue;
}
if Self::is_multi_instance(&pname, sub_table) {
for (name, inst_val) in sub_table {
let instance_name = format!("{pname}.{name}");
let mut config = inst_val.clone();
let mut provenance = seed_user_provenance(&config);
resolve_instance_defaults(&pname, &mut config, &mut provenance);
instances.push(ProcessorInstance {
instance_name,
pname: pname.clone(),
config_toml: config,
provenance,
});
}
} else {
let mut config = val.clone();
let mut provenance = seed_user_provenance(&config);
resolve_instance_defaults(&pname, &mut config, &mut provenance);
instances.push(ProcessorInstance {
instance_name: pname.clone(),
pname,
config_toml: config,
provenance,
});
}
}
}
Self { instances }
}
fn is_multi_instance(type_name: &str, table: &toml::map::Map<String, toml::Value>) -> bool {
matches!(
Self::classify_section(type_name, table),
SectionShape::MultiInstance
)
}
pub(super) fn classify_section(
type_name: &str,
table: &toml::map::Map<String, toml::Value>,
) -> SectionShape {
if table.is_empty() {
return SectionShape::SingleInstance;
}
let Some(known) = Self::known_fields_for(type_name) else {
return SectionShape::SingleInstance;
};
let known_fields: Vec<&str> = known
.iter()
.chain(SCAN_CONFIG_FIELDS.iter())
.chain(STANDARD_EXTRA_FIELDS.iter())
.copied()
.collect();
let field_keys: Vec<&str> = table
.keys()
.filter(|k| known_fields.contains(&k.as_str()))
.map(String::as_str)
.collect();
let all_values_are_tables = table.values().all(toml::Value::is_table);
match (field_keys.is_empty(), all_values_are_tables) {
(true, true) => SectionShape::MultiInstance,
(false, _) => {
if all_values_are_tables {
SectionShape::Ambiguous {
colliding: field_keys.iter().map(|s| (*s).to_string()).collect(),
}
} else {
SectionShape::SingleInstance
}
}
(true, false) => SectionShape::SingleInstance,
}
}
pub(crate) fn resolve_scan_defaults(&mut self) {
for inst in &mut self.instances {
resolve_instance_defaults(&inst.pname, &mut inst.config_toml, &mut inst.provenance);
}
}
pub(crate) fn apply_output_dir_defaults(&mut self, global_output_dir: &str) {
for inst in &mut self.instances {
let type_default_prefix = format!("out/{}", inst.pname);
let instance_prefix = format!("{}/{}", global_output_dir, inst.instance_name);
for field in &["output_dir", "output"] {
if matches!(
inst.provenance.get(*field),
Some(
FieldProvenance::UserToml { .. }
| FieldProvenance::LocalToml { .. }
| FieldProvenance::CliOverride
),
) {
continue;
}
let Some(val) = inst
.config_toml
.get(field)
.and_then(|v| v.as_str())
.map(std::string::ToString::to_string)
else {
continue;
};
let type_rest = val
.strip_prefix(&type_default_prefix)
.filter(|r| r.is_empty() || r.starts_with('/'));
let new_val = if inst.instance_name != inst.pname
&& let Some(rest) = type_rest
{
format!("{instance_prefix}{rest}")
} else if global_output_dir != "out"
&& let Some(rest) = val.strip_prefix("out/")
{
format!("{global_output_dir}/{rest}")
} else {
continue;
};
if let Some(table) = inst.config_toml.as_table_mut() {
table.insert(field.to_string(), toml::Value::String(new_val));
}
inst.provenance
.insert((*field).to_string(), FieldProvenance::OutputDirDefault);
}
}
}
pub(crate) fn first_instance_of_type(&self, type_name: &str) -> Option<&ProcessorInstance> {
self.instances.iter().find(|i| i.pname == type_name)
}
pub(crate) fn instance_config_or_default<C: serde::de::DeserializeOwned>(
&self,
type_name: &str,
) -> Result<C> {
if let Some(inst) = self.first_instance_of_type(type_name) {
return inst
.config_toml
.clone()
.try_into()
.with_context(|| format!("Failed to parse [{type_name}] config"));
}
let mut value = toml::Value::Table(toml::map::Map::new());
let mut provenance = ProvenanceMap::new();
crate::registries::processor::apply_all_defaults(type_name, &mut value, &mut provenance);
value
.try_into()
.with_context(|| format!("Failed to build default config for processor '{type_name}'"))
}
pub(crate) fn instance_field_str(&self, type_name: &str, field: &str) -> Option<String> {
self.first_instance_of_type(type_name)
.and_then(|inst| inst.config_toml.get(field))
.and_then(|v| v.as_str())
.map(std::string::ToString::to_string)
}
}