specfold_core/export/
markdown.rs

1//! The built-in `markdown` exporter: a plain Markdown tree laid out for one target (Obsidian,
2//! MkDocs, Docusaurus or a GitHub wiki). `[[references]]` become that target's links; a
3//! reference that leads nowhere in the tree (unresolved, in a remote, or to a document left out)
4//! is written as plain text and reported. The output directory must be empty or a previous
5//! export, recognised by its manifest, whose files alone are replaced.
6
7use super::links::{self, Link};
8use crate::corpus::Corpus;
9use crate::graph::Graph;
10use crate::markdown::{body, frontmatter};
11use crate::model::{Tier, stamp::slugify};
12use anyhow::Result;
13use serde::Serialize;
14use serde_json::{Value, json};
15use std::collections::{BTreeMap, HashMap, HashSet};
16use std::path::Path;
17
18mod out;
19pub use out::check_out;
20
21pub const MANIFEST: &str = ".specfold-export.json";
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
24#[serde(rename_all = "kebab-case")]
25pub enum Target {
26    Obsidian,
27    Mkdocs,
28    Docusaurus,
29    GithubWiki,
30}
31
32impl Target {
33    pub const ALL: [&str; 4] = ["obsidian", "mkdocs", "docusaurus", "github-wiki"];
34
35    pub fn parse(s: &str) -> Option<Self> {
36        Some(match s {
37            "obsidian" => Target::Obsidian,
38            "mkdocs" => Target::Mkdocs,
39            "docusaurus" => Target::Docusaurus,
40            "github-wiki" => Target::GithubWiki,
41            _ => return None,
42        })
43    }
44
45    pub fn name(self) -> &'static str {
46        match self {
47            Target::Obsidian => "obsidian",
48            Target::Mkdocs => "mkdocs",
49            Target::Docusaurus => "docusaurus",
50            Target::GithubWiki => "github-wiki",
51        }
52    }
53
54    fn flat(self) -> bool {
55        self == Target::GithubWiki
56    }
57
58    /// Whether headings keep their `{#anchor}` (MkDocs with attr_list, Docusaurus).
59    fn keeps_anchors(self) -> bool {
60        matches!(self, Target::Mkdocs | Target::Docusaurus)
61    }
62}
63
64#[derive(Debug, Clone)]
65pub struct Options {
66    pub target: Target,
67    /// Include documents in the git-ignored local directory.
68    pub local: bool,
69    /// Start each page with a note naming its source file.
70    pub banner: bool,
71}
72
73/// A reference written as plain text because it leads nowhere in the tree.
74#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
75pub struct Unresolved {
76    pub path: String,
77    pub raw: String,
78    pub reason: String,
79}
80
81#[derive(Debug, Clone, Default)]
82pub struct Tree {
83    /// Relative path and content, in path order.
84    pub files: BTreeMap<String, String>,
85    pub unresolved: Vec<Unresolved>,
86    pub docs: usize,
87}
88
89struct Page {
90    idx: usize,
91    /// Path in the tree, e.g. `shop/adr/cart-totals.md`.
92    file: String,
93    title: String,
94    /// Body with chapters appended, and its resolved references.
95    body: String,
96    links: Vec<Link>,
97    /// Explicit anchor -> heading text.
98    headings: HashMap<String, String>,
99}
100
101/// Characters no target or file system accepts in a file name.
102fn safe_name(s: &str) -> String {
103    let out: String =
104        s.chars().map(|c| if c.is_control() || "\\/:*?\"<>|#^[]".contains(c) { '-' } else { c }).collect();
105    let out = out.trim_matches(|c: char| c == '-' || c == '.' || c.is_whitespace()).to_string();
106    if out.is_empty() { "untitled".into() } else { out }
107}
108
109fn base_name(c: &Corpus, idx: usize) -> String {
110    let d = &c.docs[idx];
111    if !d.slug.is_empty() {
112        return safe_name(&d.slug);
113    }
114    let t = d.title.clone().unwrap_or_default();
115    let s = slugify(&t);
116    if !s.is_empty() {
117        return s;
118    }
119    if !t.trim().is_empty() {
120        return safe_name(&t);
121    }
122    safe_name(&d.id.key)
123}
124
125/// The text of every file of a document, main body first, frontmatter removed.
126fn full_body(c: &Corpus, idx: usize) -> String {
127    let d = &c.docs[idx];
128    let read = |p: &str| crate::corpus::overlay::read(&c.cfg.root.join(p)).unwrap_or_default();
129    let mut out = frontmatter::split(&read(&d.path)).body.trim_start_matches(['\n', '\r']).to_string();
130    let mut chapters = d.chapters.clone();
131    chapters.sort();
132    for ch in chapters {
133        if !out.ends_with('\n') {
134            out.push('\n');
135        }
136        out.push('\n');
137        out.push_str(frontmatter::split(&read(&ch)).body);
138    }
139    out
140}
141
142/// Builds the tree in memory; nothing is written.
143pub fn build(c: &Corpus, g: &Graph, o: &Options) -> Tree {
144    let mut picked: Vec<usize> = (0..c.docs.len()).filter(|&i| o.local || c.docs[i].tier == Tier::Committed).collect();
145    picked.sort_by_key(|&i| c.docs[i].id.to_string());
146    let mut taken: HashSet<String> = HashSet::new();
147    for reserved in ["home.md", "_sidebar.md", "index.md"] {
148        taken.insert(reserved.into());
149    }
150    let mut pages: Vec<Page> = Vec::new();
151    let mut by_idx: HashMap<usize, usize> = HashMap::new();
152    for &idx in &picked {
153        let d = &c.docs[idx];
154        let dir = if o.target.flat() { String::new() } else { format!("{}/{}/", safe_name(&d.id.owner), d.id.ty) };
155        let base = base_name(c, idx);
156        let mut file = format!("{dir}{base}.md");
157        let mut n = 2;
158        while !taken.insert(file.to_lowercase()) {
159            file = format!("{dir}{base}-{n}.md");
160            n += 1;
161        }
162        let text = full_body(c, idx);
163        let scanned = body::scan(&text, 1, &body::path_cite_re(".spec"));
164        let headings = scanned.anchors.into_iter().map(|a| (a.name, a.heading)).collect();
165        let links = links::of_body(c, g, &text);
166        by_idx.insert(idx, pages.len());
167        pages.push(Page {
168            idx,
169            file,
170            title: d.title.clone().unwrap_or_else(|| d.id.to_string()),
171            body: text,
172            links,
173            headings,
174        });
175    }
176    let mut tree = Tree { docs: pages.len(), ..Default::default() };
177    // Relation targets are looked up per page; a scan of every document each time made export
178    // quadratic. The first document with an id wins, as the scan did.
179    let mut by_id: HashMap<String, usize> = HashMap::new();
180    for (i, d) in c.docs.iter().enumerate() {
181        by_id.entry(d.id.to_string()).or_insert(i);
182    }
183    let ix = Index { pages: &pages, by_idx: &by_idx, by_id: &by_id };
184    for p in &pages {
185        let mut out = String::new();
186        if !o.target.flat() {
187            out.push_str(&front(c, p, o.target, &ix));
188        }
189        let d = &c.docs[p.idx];
190        if o.banner {
191            out.push_str(&format!("> Exported by specfold from `{}`. Edit the source, not this file.\n\n", d.path));
192        }
193        if let Some(s) = d.status().filter(|s| s == "superseded" || s == "rejected") {
194            out.push_str(&format!("> **Status: {s}**\n\n"));
195        }
196        out.push_str(&rewrite(p, o.target, &pages, &by_idx, &mut tree.unresolved, &d.path));
197        tree.files.insert(p.file.clone(), out);
198    }
199    index_files(&mut tree, c, o.target, &pages);
200    tree
201}
202
203fn yaml_str(s: &str) -> String {
204    serde_json::to_string(s).expect("serializing a str cannot fail")
205}
206
207fn yaml_list(v: &[String]) -> String {
208    format!("[{}]", v.iter().map(|s| yaml_str(s)).collect::<Vec<_>>().join(", "))
209}
210
211fn strings(v: Option<&Value>) -> Vec<String> {
212    match v {
213        Some(Value::String(s)) => vec![s.clone()],
214        Some(Value::Array(a)) => a.iter().filter_map(|x| x.as_str().map(str::to_string)).collect(),
215        _ => vec![],
216    }
217}
218
219/// Lookups shared by every exported page.
220struct Index<'a> {
221    pages: &'a [Page],
222    by_idx: &'a HashMap<usize, usize>,
223    by_id: &'a HashMap<String, usize>,
224}
225
226fn front(c: &Corpus, p: &Page, t: Target, ix: &Index) -> String {
227    let d = &c.docs[p.idx];
228    let fm = d.fm();
229    let get = |k: &str| fm.and_then(|m| m.get(k));
230    let mut lines = vec!["---".to_string()];
231    if t == Target::Docusaurus {
232        let stem = p.file.rsplit('/').next().expect("rsplit yields at least one item").trim_end_matches(".md");
233        lines.push(format!("id: {}", yaml_str(stem)));
234        lines.push(format!("specfold_id: {}", yaml_str(&d.id.to_string())));
235        lines.push(format!("sidebar_label: {}", yaml_str(&p.title)));
236        lines.push(format!("sidebar_position: {}", ix.by_idx[&p.idx] + 1));
237    } else {
238        lines.push(format!("id: {}", yaml_str(&d.id.to_string())));
239    }
240    lines.push(format!("type: {}", yaml_str(&d.id.ty)));
241    if let Some(s) = d.status() {
242        lines.push(format!("status: {}", yaml_str(&s)));
243    }
244    lines.push(format!("title: {}", yaml_str(&p.title)));
245    for k in ["aliases", "tags"] {
246        let v = strings(get(k));
247        if !v.is_empty() {
248            lines.push(format!("{k}: {}", yaml_list(&v)));
249        }
250    }
251    for k in ["created", "updated"] {
252        if let Some(v) = get(k).and_then(|v| v.as_str()) {
253            lines.push(format!("{k}: {}", yaml_str(v)));
254        }
255    }
256    if let Some(Value::Object(rel)) = get("rel") {
257        for (kind, v) in rel {
258            let ids: Vec<String> = strings(Some(v))
259                .into_iter()
260                .map(|s| match (t, g_target(&s, ix)) {
261                    (Target::Obsidian, Some(q)) => format!("[[{}|{}]]", q.file.trim_end_matches(".md"), q.title),
262                    _ => s,
263                })
264                .collect();
265            if !ids.is_empty() {
266                lines.push(format!("rel_{}: {}", kind.replace(['-', '.'], "_"), yaml_list(&ids)));
267            }
268        }
269    }
270    lines.push("---".into());
271    lines.join("\n") + "\n\n"
272}
273
274/// The exported page for a relation target written as an id.
275fn g_target<'a>(id: &str, ix: &Index<'a>) -> Option<&'a Page> {
276    let target = crate::model::RefText::parse(id).target;
277    let idx = ix.by_id.get(target.as_str())?;
278    ix.by_idx.get(idx).map(|&i| &ix.pages[i])
279}
280
281/// GitHub's heading slug: lower case, punctuation dropped, spaces to `-`.
282pub fn github_slug(heading: &str) -> String {
283    heading
284        .trim()
285        .to_lowercase()
286        .chars()
287        .filter_map(|c| {
288            if c.is_alphanumeric() || c == '-' || c == '_' {
289                Some(c)
290            } else if c == ' ' {
291                Some('-')
292            } else {
293                None
294            }
295        })
296        .collect()
297}
298
299/// A relative link from `from` to `to`, both tree paths.
300fn relative(from: &str, to: &str) -> String {
301    let from_dir: Vec<&str> = from.split('/').collect::<Vec<_>>()[..from.matches('/').count()].to_vec();
302    let to_parts: Vec<&str> = to.split('/').collect();
303    let common = from_dir.iter().zip(&to_parts).take_while(|(a, b)| a == b).count();
304    let mut out: Vec<String> = vec!["..".into(); from_dir.len() - common];
305    out.extend(to_parts[common..].iter().map(|s| s.to_string()));
306    out.join("/")
307}
308
309fn rewrite(
310    p: &Page,
311    t: Target,
312    pages: &[Page],
313    by_idx: &HashMap<usize, usize>,
314    unresolved: &mut Vec<Unresolved>,
315    src: &str,
316) -> String {
317    let mut out = String::new();
318    let mut at = 0;
319    for l in &p.links {
320        out.push_str(&p.body[at..l.span.0]);
321        at = l.span.1;
322        let dest = l.idx.and_then(|i| by_idx.get(&i)).map(|&i| &pages[i]);
323        let text = l.label.clone().or_else(|| dest.map(|q| q.title.clone())).or_else(|| l.title.clone());
324        let Some(q) = dest.filter(|_| l.reason.is_none()) else {
325            let reason = l.reason.clone().unwrap_or_else(|| {
326                if l.remote.is_some() {
327                    format!("in remote {}", l.remote.clone().unwrap_or_default())
328                } else {
329                    "not part of this export".into()
330                }
331            });
332            unresolved.push(Unresolved { path: src.into(), raw: l.raw.clone(), reason });
333            let inner = l.raw.trim_start_matches("[[").trim_end_matches("]]");
334            out.push_str(&text.unwrap_or_else(|| inner.split('|').next().unwrap_or(inner).trim().to_string()));
335            continue;
336        };
337        let text = text.unwrap_or_else(|| q.title.clone());
338        let heading = l.anchor.as_ref().map(|a| q.headings.get(a).cloned().unwrap_or_else(|| a.clone()));
339        out.push_str(&match t {
340            Target::Obsidian => {
341                let h = heading.map(|h| format!("#{h}")).unwrap_or_default();
342                format!("[[{}{h}|{text}]]", q.file.trim_end_matches(".md"))
343            }
344            Target::Mkdocs | Target::Docusaurus => {
345                let a = l.anchor.as_ref().map(|a| format!("#{a}")).unwrap_or_default();
346                format!("[{text}]({}{a})", relative(&p.file, &q.file))
347            }
348            Target::GithubWiki => {
349                let a = heading.map(|h| format!("#{}", github_slug(&h))).unwrap_or_default();
350                format!("[{text}]({}{a})", q.file.trim_end_matches(".md"))
351            }
352        });
353    }
354    out.push_str(&p.body[at..]);
355    if t.keeps_anchors() { out } else { strip_anchors(&out) }
356}
357
358/// Removes `{#anchor}` from headings outside code fences.
359fn strip_anchors(text: &str) -> String {
360    let re = regex::Regex::new(r"\s*\{#[A-Za-z0-9][A-Za-z0-9_.-]*\}\s*$").expect("static regex is valid");
361    let mut fence = crate::markdown::Fence::default();
362    let mut out = String::new();
363    for line in text.split_inclusive('\n') {
364        let tr = line.trim_start();
365        fence.step(tr);
366        if !fence.inside() && tr.starts_with('#') {
367            let nl = if line.ends_with('\n') { "\n" } else { "" };
368            out.push_str(&re.replace(line.trim_end_matches(['\n', '\r']), ""));
369            out.push_str(nl);
370        } else {
371            out.push_str(line);
372        }
373    }
374    out
375}
376
377fn index_files(tree: &mut Tree, c: &Corpus, t: Target, pages: &[Page]) {
378    let mut by_type: BTreeMap<&str, Vec<&Page>> = BTreeMap::new();
379    for p in pages {
380        by_type.entry(c.docs[p.idx].id.ty.as_str()).or_default().push(p);
381    }
382    let list = |link: &dyn Fn(&Page) -> String| {
383        let mut s = String::new();
384        for (ty, ps) in &by_type {
385            s.push_str(&format!("\n## {ty}\n\n"));
386            for p in ps {
387                s.push_str(&format!("- {}\n", link(p)));
388            }
389        }
390        s
391    };
392    match t {
393        Target::GithubWiki => {
394            let link = |p: &Page| format!("[{}]({})", p.title, p.file.trim_end_matches(".md"));
395            tree.files.insert("Home.md".into(), format!("# Specs\n{}", list(&link)));
396            tree.files.insert("_Sidebar.md".into(), list(&link).trim_start().to_string());
397        }
398        Target::Mkdocs | Target::Docusaurus => {
399            let link = |p: &Page| format!("[{}]({})", p.title, p.file);
400            tree.files.insert("index.md".into(), format!("# Specs\n{}", list(&link)));
401        }
402        Target::Obsidian => {}
403    }
404}
405
406/// Whether `out` may be written: missing, empty, or holding a previous export's manifest.
407/// Returns the files the previous export wrote.
408/// Writes the tree and its manifest, removing the files of the previous export first.
409pub fn write(out: &Path, tree: &Tree, target: Target, previous: &[String]) -> Result<()> {
410    for f in previous {
411        if f.split('/').any(|s| s == ".." || s.is_empty()) || Path::new(f).is_absolute() {
412            continue;
413        }
414        let p = out.join(f);
415        if p.is_file() {
416            std::fs::remove_file(&p)?;
417            let mut dir = p.parent();
418            while let Some(d) = dir.filter(|d| *d != out) {
419                if std::fs::remove_dir(d).is_err() {
420                    break;
421                }
422                dir = d.parent();
423            }
424        }
425    }
426    for (rel, text) in &tree.files {
427        let p = out.join(rel);
428        std::fs::create_dir_all(p.parent().expect("an output path joined from a file name has a parent"))?;
429        std::fs::write(&p, text)?;
430    }
431    let manifest = json!({
432        "generator": "specfold",
433        "target": target.name(),
434        "files": tree.files.keys().collect::<Vec<_>>(),
435    });
436    // C222: an export of no documents is the manifest alone, in a folder made for it.
437    std::fs::create_dir_all(out)?;
438    std::fs::write(out.join(MANIFEST), serde_json::to_string_pretty(&manifest)? + "\n")?;
439    Ok(())
440}
441
442#[cfg(test)]
443mod tests {
444    use super::*;
445
446    #[test]
447    fn names_and_paths() {
448        assert_eq!(safe_name("a:b/c"), "a-b-c");
449        assert_eq!(relative("shop/adr/a.md", "shop/sot/b.md"), "../sot/b.md");
450        assert_eq!(relative("shop/adr/a.md", "shop/adr/b.md"), "b.md");
451        assert_eq!(github_slug("Cart Totals (v2)"), "cart-totals-v2");
452        assert_eq!(strip_anchors("## Totals {#CO-1}\n```\n## x {#y}\n```\n"), "## Totals\n```\n## x {#y}\n```\n");
453    }
454}
specfold_core/export/markdown.rs

specfold_core/export/
markdown.rs
