dmc/engine/
collection.rs

1use dmc_diagnostic::Code;
2use duck_diagnostic::{DiagnosticEngine, diag};
3use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
4use std::path::PathBuf;
5
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8
9use crate::engine::{
10  cache::{FileCache, fingerprint},
11  compile::Compiler,
12  config::EngineConfig,
13  sidecar::run_sidecar,
14  utils::{CollectionReport, build_schema_ctx, build_velite_record, minify_js, wrap_mdx_module},
15};
16
17#[derive(Debug, Clone, Serialize, Deserialize, Default)]
18#[serde(default)]
19pub struct Collection {
20  pub name: String,
21  pub pattern: String,
22  pub base_dir: PathBuf,
23  #[serde(skip_serializing_if = "Option::is_none")]
24  pub schema: Option<Value>,
25  #[serde(skip_serializing_if = "std::ops::Not::not")]
26  pub single: bool,
27}
28
29impl Collection {
30  /// Compile every file matched by `pattern` in parallel, validate
31  /// frontmatter against `schema`, optionally run JS sidecars + MDX module
32  /// wrap + minify, then write `{name}.json`.
33  pub(crate) fn process(
34    &self,
35    cfg: &EngineConfig,
36    diag_engine: &mut DiagnosticEngine<Code>,
37  ) -> Result<CollectionReport, ()> {
38    let walker = globwalk::GlobWalkerBuilder::from_patterns(&self.base_dir, &[&self.pattern]).build().map_err(|e| {
39      diag_engine.emit(diag!(Code::IoRead, format!("globwalk {}: {}", self.pattern, e)));
40    })?;
41
42    let paths = walker.filter_map(|e| e.ok()).map(|e| e.path().to_path_buf()).collect::<Vec<PathBuf>>();
43
44    let collection_schema = self.schema.as_ref().and_then(|d| {
45      dmc_schema::compile_descriptor(d)
46        .map_err(|e| {
47          diag_engine.emit(diag!(Code::JsonDeserialize, format!("schema descriptor for `{}`: {}", self.name, e)));
48        })
49        .ok()
50    });
51
52    // Persistent per-file cache. Each record is keyed by
53    // (dmc_version, source_bytes, path, full-cfg-fingerprint) so any
54    // change to source or relevant config invalidates the entry.
55    let cache = if cfg.cache_enabled { FileCache::open(cfg.output_dir.join(".cache").join("dmc")) } else { None };
56    let cfg_fp = fingerprint(&(&cfg.compile, &cfg.include_html, &self.name, &self.schema, &cfg.output_format));
57
58    let outcomes: Vec<Option<Value>> = paths
59      .par_iter()
60      .map(|path| {
61        let mut local_diag_engine = DiagnosticEngine::<Code>::new();
62
63        let source = match std::fs::read_to_string(path) {
64          Ok(s) => s,
65          Err(e) => {
66            local_diag_engine.emit(diag!(Code::IoRead, format!("read source at {}: {}", path.display(), e)));
67            local_diag_engine.print_all_compact();
68            return None;
69          },
70        };
71
72        // Cache lookup: skip lex/parse/transform/codegen + sidecar when
73        // (source + cfg) is unchanged. Hits the disk and returns the
74        // already-rendered Value directly.
75        let cache_key = cache.as_ref().map(|_| FileCache::key(source.as_bytes(), path, &cfg_fp));
76        if let (Some(c), Some(k)) = (cache.as_ref(), cache_key.as_ref())
77          && let Some(hit) = c.get(k)
78        {
79          local_diag_engine.print_all(&source);
80          return Some(hit);
81        }
82
83        let local_compiler_cfg = cfg.compile.for_render();
84        let use_sidecar = cfg.compile.has_js_plugins();
85
86        let mut compiled = Compiler::compile_with_pipeline(&source, path, &local_compiler_cfg, &mut local_diag_engine);
87
88        if use_sidecar && let Some(html) = run_sidecar(&compiled.content, cfg) {
89          compiled.html = html;
90        }
91
92        if cfg.compile.mdx_output_format.as_deref() == Some("module") {
93          compiled.body = wrap_mdx_module(&compiled.body, &compiled.imports);
94        }
95        if cfg.compile.mdx_minify {
96          compiled.body = minify_js(&compiled.body);
97        }
98
99        let validated_frontmatter = match (&collection_schema, &compiled.frontmatter) {
100          (Some(schema), fm) if !fm.is_null() => {
101            let ctx = build_schema_ctx(path, &cfg.root, &compiled, cfg);
102            match schema.parse(fm, &ctx) {
103              Ok(v) => v,
104              Err(e) => {
105                local_diag_engine
106                  .emit(diag!(Code::JsonDeserialize, format!("frontmatter validation at {}: {}", path.display(), e)));
107                compiled.frontmatter.clone()
108              },
109            }
110          },
111          _ => compiled.frontmatter.clone(),
112        };
113
114        let include_html = cfg.include_html || use_sidecar;
115        let rec = build_velite_record(compiled, validated_frontmatter, path, &self.base_dir, &self.name, include_html);
116
117        // Only cache clean runs. A run that produced errors (lex, parse,
118        // transform, frontmatter validation) re-runs every build until
119        // the source is fixed, so the user sees the diagnostics every
120        // time instead of getting a silent cache hit on poisoned output.
121        let dirty = local_diag_engine.error_count() + local_diag_engine.bug_count() > 0;
122        if !dirty && let (Some(c), Some(k)) = (cache.as_ref(), cache_key.as_ref()) {
123          c.put(k, &rec);
124        }
125        local_diag_engine.print_all(&source);
126
127        Some(rec)
128      })
129      .collect();
130
131    let mut records: Vec<Value> = Vec::with_capacity(outcomes.len());
132    for r in outcomes.into_iter().flatten() {
133      // diag_engine.extend(local_diag_engine);
134      records.push(r);
135    }
136
137    let out_path = cfg.output_dir.join(format!("{}.json", self.name));
138    let count = if self.single { if records.is_empty() { 0 } else { 1 } } else { records.len() };
139    let json = if self.single {
140      let single = records.into_iter().next().unwrap_or(Value::Null);
141      serde_json::to_string_pretty(&single).unwrap()
142    } else {
143      serde_json::to_string_pretty(&records).unwrap()
144    };
145
146    std::fs::write(&out_path, json).map_err(|e| {
147      diag_engine.emit(diag!(Code::IoWrite, format!("collection {} write at {}: {}", self.name, out_path.display(), e)))
148    })?;
149
150    Ok(CollectionReport { name: self.name.clone(), records: count, output_path: out_path })
151  }
152}
dmc/engine/collection.rs

dmc/engine/
collection.rs
