dmc/engine/
collection.rs

1use dmc_diagnostic::Code;
2use duck_diagnostic::{DiagnosticEngine, diag};
3use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
4use std::path::PathBuf;
5
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8
9use crate::engine::{
10  cache::{FileCache, fingerprint},
11  compile::Compiler,
12  config::EngineConfig,
13  sidecar::run_sidecar,
14  utils::{CollectionReport, build_schema_ctx, build_velite_record, minify_js, wrap_mdx_module},
15};
16
17#[derive(Debug, Clone, Serialize, Deserialize, Default)]
18#[serde(default)]
19pub struct Collection {
20  pub name: String,
21  pub pattern: String,
22  pub base_dir: PathBuf,
23  #[serde(skip_serializing_if = "Option::is_none")]
24  pub schema: Option<Value>,
25  #[serde(skip_serializing_if = "std::ops::Not::not")]
26  pub single: bool,
27}
28
29impl Collection {
30  /// Compile matched files in parallel, validate frontmatter, optionally
31  /// run JS sidecars + MDX module wrap + minify, write `{name}.json`.
32  pub(crate) fn process(
33    &self,
34    cfg: &EngineConfig,
35    diag_engine: &mut DiagnosticEngine<Code>,
36  ) -> Result<CollectionReport, ()> {
37    let walker = globwalk::GlobWalkerBuilder::from_patterns(&self.base_dir, &[&self.pattern]).build().map_err(|e| {
38      diag_engine.emit(diag!(Code::IoRead, format!("globwalk {}: {}", self.pattern, e)));
39    })?;
40
41    let paths = walker.filter_map(|e| e.ok()).map(|e| e.path().to_path_buf()).collect::<Vec<PathBuf>>();
42
43    let collection_schema = self.schema.as_ref().and_then(|d| {
44      dmc_schema::compile_descriptor(d)
45        .map_err(|e| {
46          diag_engine.emit(diag!(Code::JsonDeserialize, format!("schema descriptor for `{}`: {}", self.name, e)));
47        })
48        .ok()
49    });
50
51    let cache = if cfg.cache_enabled { FileCache::open(cfg.output_dir.join(".cache").join("dmc")) } else { None };
52    let cfg_fp = fingerprint(&(&cfg.compile, &cfg.include_html, &self.name, &self.schema, &cfg.output_format));
53
54    let outcomes: Vec<Option<Value>> = paths
55      .par_iter()
56      .map(|path| {
57        let mut local_diag_engine = DiagnosticEngine::<Code>::new();
58
59        let source = match std::fs::read_to_string(path) {
60          Ok(s) => s,
61          Err(e) => {
62            local_diag_engine.emit(diag!(Code::IoRead, format!("read source at {}: {}", path.display(), e)));
63            local_diag_engine.print_all_compact();
64            return None;
65          },
66        };
67
68        let cache_key = cache.as_ref().map(|_| FileCache::key(source.as_bytes(), path, &cfg_fp));
69        if let (Some(c), Some(k)) = (cache.as_ref(), cache_key.as_ref())
70          && let Some(hit) = c.get(k)
71        {
72          local_diag_engine.print_all(&source);
73          return Some(hit);
74        }
75
76        let local_compiler_cfg = cfg.compile.for_render();
77        let use_sidecar = cfg.compile.has_js_plugins();
78
79        let mut compiled = Compiler::compile_with_pipeline(&source, path, &local_compiler_cfg, &mut local_diag_engine);
80
81        if use_sidecar && let Some(html) = run_sidecar(&compiled.content, cfg) {
82          compiled.html = html;
83        }
84
85        if cfg.compile.mdx_output_format.as_deref() == Some("module") {
86          compiled.body = wrap_mdx_module(&compiled.body, &compiled.imports);
87        }
88        if cfg.compile.mdx_minify {
89          compiled.body = minify_js(&compiled.body);
90        }
91
92        let validated_frontmatter = match (&collection_schema, &compiled.frontmatter) {
93          (Some(schema), fm) if !fm.is_null() => {
94            let ctx = build_schema_ctx(path, &cfg.root, &compiled, cfg);
95            match schema.parse(fm, &ctx) {
96              Ok(v) => v,
97              Err(e) => {
98                local_diag_engine
99                  .emit(diag!(Code::JsonDeserialize, format!("frontmatter validation at {}: {}", path.display(), e)));
100                compiled.frontmatter.clone()
101              },
102            }
103          },
104          _ => compiled.frontmatter.clone(),
105        };
106
107        let include_html = cfg.include_html || use_sidecar;
108        let rec = build_velite_record(compiled, validated_frontmatter, path, &self.base_dir, &self.name, include_html);
109
110        // Cache only clean runs so diagnostics re-fire until the source is fixed.
111        let dirty = local_diag_engine.error_count() + local_diag_engine.bug_count() > 0;
112        if !dirty && let (Some(c), Some(k)) = (cache.as_ref(), cache_key.as_ref()) {
113          c.put(k, &rec);
114        }
115        local_diag_engine.print_all(&source);
116
117        Some(rec)
118      })
119      .collect();
120
121    let mut records: Vec<Value> = Vec::with_capacity(outcomes.len());
122    for r in outcomes.into_iter().flatten() {
123      records.push(r);
124    }
125
126    let out_path = cfg.output_dir.join(format!("{}.json", self.name));
127    let count = if self.single { if records.is_empty() { 0 } else { 1 } } else { records.len() };
128    let json = if self.single {
129      let single = records.into_iter().next().unwrap_or(Value::Null);
130      serde_json::to_string_pretty(&single).unwrap()
131    } else {
132      serde_json::to_string_pretty(&records).unwrap()
133    };
134
135    std::fs::write(&out_path, json).map_err(|e| {
136      diag_engine.emit(diag!(Code::IoWrite, format!("collection {} write at {}: {}", self.name, out_path.display(), e)))
137    })?;
138
139    Ok(CollectionReport { name: self.name.clone(), records: count, output_path: out_path })
140  }
141}
dmc/engine/collection.rs

dmc/engine/
collection.rs
