use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::fs;
use serde::Deserialize;
use crate::proc::{
Asset, Context, ContextValue, MediaType, ProcessesAssets, ProcessingError,
canonicalize::CanonicalizeProcessor,
favicon::FaviconProcessor,
image::ImageResizeProcessor,
js_bundle::JsBundleProcessor,
markdown::MarkdownProcessor,
minify_html::MinifyHtmlProcessor,
minify_js::MinifyJsProcessor,
scss::ScssProcessor,
template::{PART_CONTEXT_PREFIX, TemplateProcessor},
};
use crate::tool::{Config, DEFAULT_CONFIG_FILE, DEFAULT_CONFIG_PROFILE};
const PART_PATH_PREFIX: &str = "_";
pub fn is_part(path: &str) -> bool {
path.split(['/', '\\'])
.any(|component| component.starts_with(PART_PATH_PREFIX))
}
pub async fn run(procs_file: Option<&Path>, profile: Option<&str>) -> std::io::Result<()> {
let config_path = procs_file.unwrap_or(Path::new(DEFAULT_CONFIG_FILE));
let profile_name = profile.unwrap_or(DEFAULT_CONFIG_PROFILE);
let config_toml = fs::read_to_string(config_path).await?;
let config: Config = toml::from_str(&config_toml).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid TOML: {}", e),
)
})?;
let default_profile = config.profiles.get(DEFAULT_CONFIG_PROFILE).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("missing default profile: {}", DEFAULT_CONFIG_PROFILE),
)
})?;
let config = if profile_name == DEFAULT_CONFIG_PROFILE {
default_profile.clone()
} else {
let selected_profile = config.profiles.get(profile_name).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("missing selected profile: {}", profile_name),
)
})?;
default_profile.merge(selected_profile)
};
let source_path = config.paths.get("source").ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"missing paths.source in context",
)
})?;
let target_path = config.paths.get("target").ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"missing paths.target in context",
)
})?;
tracing::info!("Processing assets from {} to {}", source_path, target_path);
tracing::info!("Profile: {}", profile_name);
tracing::debug!("Processors: {:?}", config.procs.keys().collect::<Vec<_>>());
let source = Path::new(source_path);
let target = Path::new(target_path);
let mut assets = Vec::new();
collect_assets(source, &mut assets).await?;
tracing::info!("Found {} assets", assets.len());
if !fs::try_exists(target).await? {
fs::create_dir_all(target).await?;
}
let mut proc_context = Context::new();
for (key, value) in config.context {
proc_context.insert(key.clone().into(), ContextValue::Text(value.clone().into()));
}
proc_context.insert(
"_asset_source_root".into(),
ContextValue::Text(source_path.clone().into()),
);
let mut regular_assets = Vec::new();
let mut part_count = 0;
for (relative_path, content) in assets {
if is_part(&relative_path) {
let part_key = format!("{}{}", PART_CONTEXT_PREFIX, relative_path);
let content_str = String::from_utf8_lossy(&content).to_string();
proc_context.insert(part_key.into(), ContextValue::Text(content_str.into()));
part_count += 1;
tracing::debug!("Cached part: {}", relative_path);
} else {
regular_assets.push((relative_path, content));
}
}
tracing::info!("Cached {} parts", part_count);
let procs = Arc::new(config.procs);
let proc_context = Arc::new(proc_context);
let target = Arc::new(target.to_path_buf());
let handles: Vec<_> = regular_assets
.into_iter()
.map(|(relative_path, content)| {
let procs = Arc::clone(&procs);
let proc_context = Arc::clone(&proc_context);
let target = Arc::clone(&target);
tokio::spawn(async move {
let result =
process_asset(&relative_path, content, &procs, &proc_context, &target).await;
(relative_path, result)
})
})
.collect();
let mut success_count = 0;
let mut error_count = 0;
for handle in handles {
match handle.await {
Ok((_path, Ok(()))) => success_count += 1,
Ok((path, Err(e))) => {
tracing::error!("Error processing {}: {}", path, e);
error_count += 1;
}
Err(e) => {
tracing::error!("Task panicked: {}", e);
error_count += 1;
}
}
}
tracing::info!(
"Processed {} assets ({} errors)",
success_count,
error_count
);
Ok(())
}
pub async fn collect_assets(
root: &Path,
assets: &mut Vec<(String, Vec<u8>)>,
) -> std::io::Result<()> {
let mut stack: Vec<PathBuf> = vec![root.to_path_buf()];
while let Some(current) = stack.pop() {
let mut entries = fs::read_dir(¤t).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let metadata = fs::metadata(&path).await?;
if metadata.is_dir() {
stack.push(path);
} else if metadata.is_file() {
let relative = path.strip_prefix(root).map_err(std::io::Error::other)?;
let relative_str = relative.to_string_lossy().to_string();
let content = fs::read(&path).await?;
assets.push((relative_str, content));
}
}
}
Ok(())
}
const TRANSFORMATION_PROCESSORS: &[&str] = &[
"template",
"markdown",
"scss",
"js_bundle",
"image",
"favicon",
];
const FINALIZATION_PROCESSORS: &[&str] = &["canonicalize", "minify_html", "minify_js"];
pub async fn process_asset(
path: &str,
content: Vec<u8>,
procs: &BTreeMap<String, ProcessorConfig>,
context: &Context,
target: &Path,
) -> std::io::Result<()> {
let mut asset = Asset::new(path.into(), content);
let mut context = context.clone();
if let Some(config) = procs.get("canonicalize")
&& let Some(root) = &config.root
{
let target_path = if path.ends_with(".md") {
path.trim_end_matches(".md").to_string() + ".html"
} else {
path.to_string()
};
let canonical_path = format!("{}/{}", root.trim_end_matches('/'), target_path,);
context.insert("path".into(), ContextValue::Text(canonical_path.into()));
}
let pattern_enabled = procs.contains_key("pattern");
let mut ran_processors: Vec<&str> = Vec::new();
loop {
let mut processed_types: Vec<MediaType> = Vec::new();
loop {
let current_type = asset.media_type().clone();
if processed_types.contains(¤t_type) {
break;
}
processed_types.push(current_type.clone());
for proc_name in TRANSFORMATION_PROCESSORS {
if let Some(config) = procs.get(*proc_name) {
let (modified, result) =
run_processor(proc_name, config, &mut context, &mut asset);
if let Err(e) = result {
tracing::warn!("Processor `{}` failed on {}: {:?}", proc_name, path, e);
} else if modified {
ran_processors.push(proc_name);
}
}
}
if asset.media_type() == ¤t_type {
break;
}
}
if pattern_enabled
&& let Some(ContextValue::Text(pattern_path)) = context.remove(&"pattern".into())
{
let content = asset.as_text().map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"pattern wrapping requires text content",
)
})?;
context.insert("content".into(), ContextValue::Text(content.clone()));
let part_key: codas::types::Text =
format!("{}{}", PART_CONTEXT_PREFIX, pattern_path).into();
let pattern_content = match context.get(&part_key) {
Some(ContextValue::Text(content)) => content.clone(),
_ => {
tracing::warn!("Pattern not found: {}", pattern_path);
break;
}
};
let pattern_media_type = pattern_path
.rsplit('.')
.next()
.map(MediaType::from_extension)
.unwrap_or(MediaType::Html);
ran_processors.push("pattern");
asset = Asset::new(path.into(), pattern_content.as_bytes().to_vec());
asset.set_media_type(pattern_media_type);
continue;
}
break;
}
for proc_name in FINALIZATION_PROCESSORS {
if let Some(config) = procs.get(*proc_name) {
let (modified, result) = run_processor(proc_name, config, &mut context, &mut asset);
if let Err(e) = result {
tracing::warn!("Processor `{}` failed on {}: {:?}", proc_name, path, e);
} else if modified {
ran_processors.push(proc_name);
}
}
}
let new_extension = asset
.media_type()
.extensions()
.first()
.expect("all media types have at least one extension");
let processed_path = if let Some(dot_pos) = path.rfind('.') {
format!("{}.{}", &path[..dot_pos], new_extension)
} else {
format!("{}.{}", path, new_extension)
};
let target_path = target.join(&processed_path);
if let Some(parent) = target_path.parent() {
fs::create_dir_all(parent).await?;
}
fs::write(&target_path, asset.as_bytes()).await?;
let source_dir = path.rsplit_once('/').map(|(dir, _)| dir);
let target_dir = processed_path.rsplit_once('/').map(|(dir, _)| dir);
let target_filename = processed_path.rsplit('/').next().unwrap_or(&processed_path);
let display_target = if source_dir.is_some() && source_dir == target_dir {
format!("/../{}", target_filename)
} else {
format!("/{}", processed_path)
};
if ran_processors.is_empty() {
tracing::debug!("COPY /{} -> {}", path, display_target);
} else {
tracing::debug!(
"PROC /{} -> [{}] -> {}",
path,
ran_processors.join(", "),
display_target
);
}
Ok(())
}
pub fn run_processor(
name: &str,
config: &ProcessorConfig,
context: &mut Context,
asset: &mut Asset,
) -> (bool, Result<(), ProcessingError>) {
let before_type = asset.media_type().clone();
let before_len = asset.as_bytes().len();
let result = match name {
"markdown" => MarkdownProcessor {}.process(context, asset),
"template" => TemplateProcessor.process(context, asset),
"favicon" => FaviconProcessor.process(context, asset),
"canonicalize" => {
let root = config.root.as_deref().unwrap_or("http://localhost/");
if let Some(processor) = CanonicalizeProcessor::new(root) {
processor.process(context, asset)
} else {
Err(ProcessingError::Malformed {
message: format!("invalid root URL: {}", root).into(),
})
}
}
"scss" => ScssProcessor {}.process(context, asset),
"js_bundle" => {
let minify = config.minify.unwrap_or(false);
JsBundleProcessor::new(minify).process(context, asset)
}
"minify_html" => MinifyHtmlProcessor.process(context, asset),
"minify_js" => MinifyJsProcessor.process(context, asset),
"image" => {
let width = config.max_width.unwrap_or(1920);
let height = config.max_height.unwrap_or(1920);
ImageResizeProcessor::new(width, height).process(context, asset)
}
_ => {
tracing::warn!("Unknown processor: {}", name);
Ok(())
}
};
let modified = result.is_ok()
&& (asset.media_type() != &before_type || asset.as_bytes().len() != before_len);
(modified, result)
}
#[derive(Debug, Default, Deserialize, Clone)]
pub struct ProcessorConfig {
root: Option<String>,
minify: Option<bool>,
max_width: Option<u32>,
max_height: Option<u32>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_parts() {
assert!(is_part("_header.html"));
assert!(is_part("_parts/footer.html"));
assert!(is_part("templates/_layout.html"));
assert!(is_part("a/b/_c/d.html"));
assert!(!is_part("index.html"));
assert!(!is_part("pages/about.html"));
assert!(!is_part("my_file.html")); }
#[test]
fn merges_profiles() {
let toml = r#"
[default.paths]
source = "site/"
target = "public/"
[default.procs]
canonicalize = { root = "http://localhost/" }
js_bundle = { minify = false }
[production.paths]
target = "dist/"
[production.procs]
canonicalize = { root = "https://prod.example.com/" }
js_bundle = { minify = true }
"#;
let config: Config = toml::from_str(toml).unwrap();
let default_profile = config.profiles.get("default").unwrap();
let production_profile = config.profiles.get("production").unwrap();
let merged = default_profile.merge(production_profile);
assert_eq!(merged.paths.get("source").unwrap(), "site/");
assert_eq!(merged.paths.get("target").unwrap(), "dist/");
let canonicalize = merged.procs.get("canonicalize").unwrap();
assert_eq!(
canonicalize.root,
Some("https://prod.example.com/".to_string())
);
let js_bundle = merged.procs.get("js_bundle").unwrap();
assert_eq!(js_bundle.minify, Some(true));
}
}