use super::reports::Mirror;
use crate::codec::Format;
use crate::model::{ExceptionItem, ExceptionList, Rule};
use crate::normalize;
use elasticctl_core::{Error, ErrorKind, Result};
use serde_json::Value;
use std::path::{Path, PathBuf};
pub fn read_mirror(dir: &Path) -> Result<Mirror> {
let mut mirror = Mirror {
rules: Vec::new(),
lists: Vec::new(),
items: Vec::new(),
value_lists: Vec::new(),
has_value_lists: false,
};
let rules_path = super::rules_dir(dir);
if let Some(paths) = mirror_root_files(&rules_path)? {
for path in paths {
let body = read_regular_file(&path)?;
let (mut rules, lists, items) = decode_rule_file(&body, Format::from_path(&path))?;
mirror.rules.append(&mut rules);
for mut list in lists {
let list_items = split_items(&mut list)?;
mirror.lists.push(list);
mirror.items.extend(list_items);
}
for item in &items {
validate_top_level_item(item)?;
}
mirror.items.extend(items);
}
}
let lists_path = super::exceptions_dir(dir);
if let Some(paths) = mirror_root_files(&lists_path)? {
for path in paths {
let body = read_regular_file(&path)?;
let mut list = decode_list_file(&body, Format::from_path(&path))?;
let items = split_items(&mut list)?;
mirror.lists.push(list);
mirror.items.extend(items);
}
}
read_value_lists(dir, &mut mirror)?;
normalize::sort_rules(&mut mirror.rules);
normalize::sort_lists(&mut mirror.lists);
normalize::sort_items(&mut mirror.items);
Ok(mirror)
}
fn read_value_lists(dir: &Path, mirror: &mut Mirror) -> Result<()> {
let root = dir.join("value-lists");
let Some(paths) = mirror_root_files(&root)? else {
return Ok(());
};
mirror.has_value_lists = true;
let mut stems = std::collections::BTreeSet::new();
for path in &paths {
if matches!(
path.extension().and_then(|s| s.to_str()),
Some("yaml" | "txt")
) {
let sibling = path.with_extension(if path.extension().unwrap() == "yaml" {
"txt"
} else {
"yaml"
});
if !paths.contains(&sibling) {
return Err(Error::new(
ErrorKind::Error,
format!(
"value-list pair {} is missing {}",
path.display(),
sibling.display()
),
));
}
stems.insert(path.file_stem().unwrap().to_owned());
}
}
for stem in stems {
let yaml = root.join(format!("{}.yaml", stem.to_string_lossy()));
let text = root.join(format!("{}.txt", stem.to_string_lossy()));
let stem = stem
.to_str()
.ok_or_else(|| Error::new(ErrorKind::Error, "value-list file stem must be UTF-8"))?;
let yaml_body = read_regular_file(&yaml)?;
let text_body = read_regular_file(&text)?;
let validation = crate::value_lists::validate_local_pair(
stem,
yaml_body.as_bytes(),
Some(text_body.as_bytes()),
);
match validation.pair {
Some(pair) => mirror.value_lists.push(pair),
None => {
return Err(validation
.findings
.refusals
.into_iter()
.next()
.expect("invalid pair has a refusal"));
}
}
}
let collisions = crate::value_lists::stem_collisions(
mirror
.value_lists
.iter()
.map(|p| (p.container.id.as_str(), p.stem.as_str())),
);
if !collisions.is_empty() {
return Err(Error::new(
ErrorKind::Conflict,
format!("value-list filename collisions: {collisions:?}"),
));
}
Ok(())
}
fn validate_top_level_item(item: &ExceptionItem) -> Result<()> {
let item_id = item.item_id()?;
if item_id.is_empty() {
return Err(Error::new(
ErrorKind::Error,
"exception item field item_id must be a non-empty string",
));
}
let list_id = item.list_id()?;
if list_id.is_empty() {
return Err(Error::new(
ErrorKind::Error,
"exception item field list_id must be a non-empty string",
));
}
match item.as_map().get("namespace_type") {
None => Ok(()),
Some(Value::String(value)) if !value.is_empty() => Ok(()),
Some(_) => Err(Error::new(
ErrorKind::Error,
"exception item field namespace_type must be a non-empty string",
)),
}
}
pub fn read_local(dir: &Path) -> Result<Vec<Rule>> {
Ok(read_mirror(dir)?.rules)
}
pub(super) fn mirror_root_files(dir: &Path) -> Result<Option<Vec<PathBuf>>> {
match std::fs::symlink_metadata(dir) {
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(Error::new(
ErrorKind::Error,
format!("reading {}: {e}", dir.display()),
)),
Ok(metadata) => {
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(Error::new(
ErrorKind::Error,
format!("mirror directory {} is not a real directory", dir.display()),
));
}
Ok(Some(mirror_files(dir)?))
}
}
}
fn mirror_files(dir: &Path) -> Result<Vec<PathBuf>> {
let mut out = Vec::new();
for entry in std::fs::read_dir(dir)
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", dir.display())))?
{
if let Some(path) = mirror_entry_path(dir, entry)? {
out.push(path);
}
}
out.sort();
Ok(out)
}
fn mirror_entry_path(
dir: &Path,
entry: std::io::Result<std::fs::DirEntry>,
) -> Result<Option<PathBuf>> {
let entry = entry
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", dir.display())))?;
let path = entry.path();
if !super::is_rule_file(&path)
&& !(dir.file_name().is_some_and(|name| name == "value-lists")
&& path.extension().is_some_and(|ext| ext == "txt"))
{
return Ok(None);
}
let file_type = entry
.file_type()
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", path.display())))?;
if !file_type.is_file() {
return Err(Error::new(
ErrorKind::Error,
format!("mirror entry {} is not a regular file", path.display()),
));
}
Ok(Some(path))
}
#[cfg(unix)]
pub(super) fn read_regular_file(path: &Path) -> Result<String> {
use std::io::Read;
use std::os::unix::fs::OpenOptionsExt;
let mut file = std::fs::OpenOptions::new()
.read(true)
.custom_flags(libc::O_NOFOLLOW)
.open(path)
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", path.display())))?;
let mut body = String::new();
file.read_to_string(&mut body)
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", path.display())))?;
Ok(body)
}
#[cfg(not(unix))]
pub(super) fn read_regular_file(path: &Path) -> Result<String> {
let metadata = std::fs::symlink_metadata(path)
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", path.display())))?;
if metadata.file_type().is_symlink() {
return Err(Error::new(
ErrorKind::Error,
format!("mirror entry {} is not a regular file", path.display()),
));
}
std::fs::read_to_string(path)
.map_err(|e| Error::new(ErrorKind::Error, format!("reading {}: {e}", path.display())))
}
fn decode_rule_file(
body: &str,
format: Format,
) -> Result<(Vec<Rule>, Vec<ExceptionList>, Vec<ExceptionItem>)> {
match format {
Format::Ndjson => {
let bundle = crate::codec::decode_bundle(body)?;
Ok((bundle.rules, bundle.lists, bundle.items))
}
Format::Yaml => {
let values: Vec<Value> = serde_yaml_ng::from_str(body)
.map_err(|e| Error::new(ErrorKind::Error, format!("parsing YAML: {e}")))?;
let mut rules = Vec::new();
let mut lists = Vec::new();
let mut items = Vec::new();
for value in values {
if value.get("rule_id").is_some() {
rules.push(Rule::from_value(value)?);
} else if value.get("item_id").is_some() {
items.push(ExceptionItem::from_value(value)?);
} else if value.get("list_id").is_some() {
lists.push(ExceptionList::from_value(value)?);
} else {
return Err(Error::new(
ErrorKind::Error,
"a mirror file entry has neither rule_id, item_id, nor list_id",
));
}
}
Ok((rules, lists, items))
}
}
}
pub(crate) fn encode_rule_file(
rule: &Rule,
inline_lists: &[ExceptionList],
format: Format,
) -> Result<String> {
let mut objects = Vec::with_capacity(1 + inline_lists.len());
objects.push(rule.clone().into_value());
for list in inline_lists {
objects.push(list.clone().into_value());
}
match format {
Format::Yaml => serde_yaml_ng::to_string(&objects)
.map_err(|e| Error::new(ErrorKind::Error, format!("encoding YAML: {e}"))),
Format::Ndjson => {
let mut out = String::new();
for object in &objects {
out.push_str(
&serde_json::to_string(object).map_err(|e| {
Error::new(ErrorKind::Error, format!("encoding NDJSON: {e}"))
})?,
);
out.push('\n');
}
Ok(out)
}
}
}
pub(crate) fn encode_list_file(list: &ExceptionList, format: Format) -> Result<String> {
match format {
Format::Yaml => serde_yaml_ng::to_string(list.as_map())
.map_err(|e| Error::new(ErrorKind::Error, format!("encoding YAML: {e}"))),
Format::Ndjson => Ok(format!(
"{}\n",
serde_json::to_string(list.as_map())
.map_err(|e| Error::new(ErrorKind::Error, format!("encoding NDJSON: {e}")))?
)),
}
}
fn decode_list_file(body: &str, format: Format) -> Result<ExceptionList> {
let value = match format {
Format::Yaml => serde_yaml_ng::from_str(body)
.map_err(|e| Error::new(ErrorKind::Error, format!("parsing exception list: {e}")))?,
Format::Ndjson => {
let mut lines = body.lines().filter(|line| !line.trim().is_empty());
let line = lines
.next()
.ok_or_else(|| Error::new(ErrorKind::Error, "empty exception list file"))?;
if lines.next().is_some() {
return Err(Error::new(
ErrorKind::Error,
"an exception mirror file must contain exactly one nonblank NDJSON object",
));
}
serde_json::from_str(line.trim())
.map_err(|e| Error::new(ErrorKind::Error, format!("parsing exception list: {e}")))?
}
};
ExceptionList::from_value(value)
}
fn split_items(list: &mut ExceptionList) -> Result<Vec<ExceptionItem>> {
let items = match list.as_map_mut().remove("items") {
None => return Ok(Vec::new()),
Some(Value::Array(items)) => items,
Some(_) => {
return Err(Error::new(
ErrorKind::Error,
"exception list field items must be an array",
));
}
};
let list_id = list.list_id()?.to_string();
let namespace = list.namespace_type().to_string();
items
.into_iter()
.map(|value| {
let mut item = ExceptionItem::from_value(value)?;
item.as_map_mut()
.insert("list_id".into(), Value::String(list_id.clone()));
item.as_map_mut()
.insert("namespace_type".into(), Value::String(namespace.clone()));
Ok(item)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mirror_entry_path_reports_an_entry_read_error_naming_the_directory() {
let dir = Path::new("/mirror");
let err = mirror_entry_path(
dir,
Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"denied",
)),
)
.unwrap_err();
assert_eq!(err.kind, ErrorKind::Error);
assert!(err.message.contains("/mirror"), "{}", err.message);
}
}