specfold_core/export/markdown/
tree.rs

1//! Building the page tree, its links and front matter, and writing it out.
2
3use super::types::{MANIFEST, Options, Target, Tree, Unresolved};
4use crate::corpus::Corpus;
5use crate::export::links::{self, Link};
6use crate::graph::Graph;
7use crate::markdown::{body, frontmatter};
8use crate::model::{Tier, stamp::slugify};
9use anyhow::Result;
10use serde_json::{Value, json};
11use std::collections::{BTreeMap, HashMap, HashSet};
12use std::path::Path;
13
14struct Page {
15    idx: usize,
16    /// Path in the tree, e.g. `shop/adr/cart-totals.md`.
17    file: String,
18    title: String,
19    /// Body with chapters appended, and its resolved references.
20    body: String,
21    links: Vec<Link>,
22    /// Explicit anchor -> heading text.
23    headings: HashMap<String, String>,
24}
25
26/// Characters no target or file system accepts in a file name.
27fn safe_name(s: &str) -> String {
28    let out: String =
29        s.chars().map(|c| if c.is_control() || "\\/:*?\"<>|#^[]".contains(c) { '-' } else { c }).collect();
30    let out = out.trim_matches(|c: char| c == '-' || c == '.' || c.is_whitespace()).to_string();
31    if out.is_empty() { "untitled".into() } else { out }
32}
33
34fn base_name(c: &Corpus, idx: usize) -> String {
35    let d = &c.docs[idx];
36    if !d.slug.is_empty() {
37        return safe_name(&d.slug);
38    }
39    let t = d.title.clone().unwrap_or_default();
40    let s = slugify(&t);
41    if !s.is_empty() {
42        return s;
43    }
44    if !t.trim().is_empty() {
45        return safe_name(&t);
46    }
47    safe_name(&d.id.key)
48}
49
50/// The text of every file of a document, main body first, frontmatter removed.
51fn full_body(c: &Corpus, idx: usize) -> String {
52    let d = &c.docs[idx];
53    let read = |p: &str| crate::corpus::overlay::read(&c.cfg.root.join(p)).unwrap_or_default();
54    let mut out = frontmatter::split(&read(&d.path)).body.trim_start_matches(['\n', '\r']).to_string();
55    let mut chapters = d.chapters.clone();
56    chapters.sort();
57    for ch in chapters {
58        if !out.ends_with('\n') {
59            out.push('\n');
60        }
61        out.push('\n');
62        out.push_str(frontmatter::split(&read(&ch)).body);
63    }
64    out
65}
66
67/// Builds the tree in memory; nothing is written.
68pub fn build(c: &Corpus, g: &Graph, o: &Options) -> Tree {
69    let mut picked: Vec<usize> = (0..c.docs.len()).filter(|&i| o.local || c.docs[i].tier == Tier::Committed).collect();
70    picked.sort_by_key(|&i| c.docs[i].id.to_string());
71    let mut taken: HashSet<String> = HashSet::new();
72    for reserved in ["home.md", "_sidebar.md", "index.md"] {
73        taken.insert(reserved.into());
74    }
75    let mut pages: Vec<Page> = Vec::new();
76    let mut by_idx: HashMap<usize, usize> = HashMap::new();
77    for &idx in &picked {
78        let d = &c.docs[idx];
79        let dir = if o.target.flat() { String::new() } else { format!("{}/{}/", safe_name(&d.id.owner), d.id.ty) };
80        let base = base_name(c, idx);
81        let mut file = format!("{dir}{base}.md");
82        let mut n = 2;
83        while !taken.insert(file.to_lowercase()) {
84            file = format!("{dir}{base}-{n}.md");
85            n += 1;
86        }
87        let text = full_body(c, idx);
88        let scanned = body::scan(&text, 1, &body::path_cite_re(".spec"));
89        let headings = scanned.anchors.into_iter().map(|a| (a.name, a.heading)).collect();
90        let links = links::of_body(c, g, &text);
91        by_idx.insert(idx, pages.len());
92        pages.push(Page {
93            idx,
94            file,
95            title: d.title.clone().unwrap_or_else(|| d.id.to_string()),
96            body: text,
97            links,
98            headings,
99        });
100    }
101    let mut tree = Tree { docs: pages.len(), ..Default::default() };
102    // Relation targets are looked up per page; a scan of every document each time made export
103    // quadratic. The first document with an id wins, as the scan did.
104    let mut by_id: HashMap<String, usize> = HashMap::new();
105    for (i, d) in c.docs.iter().enumerate() {
106        by_id.entry(d.id.to_string()).or_insert(i);
107    }
108    let ix = Index { pages: &pages, by_idx: &by_idx, by_id: &by_id };
109    for p in &pages {
110        let mut out = String::new();
111        if !o.target.flat() {
112            out.push_str(&front(c, p, o.target, &ix));
113        }
114        let d = &c.docs[p.idx];
115        if o.banner {
116            out.push_str(&format!("> Exported by specfold from `{}`. Edit the source, not this file.\n\n", d.path));
117        }
118        if let Some(s) = d.status().filter(|s| s == "superseded" || s == "rejected") {
119            out.push_str(&format!("> **Status: {s}**\n\n"));
120        }
121        out.push_str(&rewrite(p, o.target, &pages, &by_idx, &mut tree.unresolved, &d.path));
122        tree.files.insert(p.file.clone(), out);
123    }
124    index_files(&mut tree, c, o.target, &pages);
125    tree
126}
127
128fn yaml_str(s: &str) -> String {
129    serde_json::to_string(s).expect("serializing a str cannot fail")
130}
131
132fn yaml_list(v: &[String]) -> String {
133    format!("[{}]", v.iter().map(|s| yaml_str(s)).collect::<Vec<_>>().join(", "))
134}
135
136fn strings(v: Option<&Value>) -> Vec<String> {
137    match v {
138        Some(Value::String(s)) => vec![s.clone()],
139        Some(Value::Array(a)) => a.iter().filter_map(|x| x.as_str().map(str::to_string)).collect(),
140        _ => vec![],
141    }
142}
143
144/// Lookups shared by every exported page.
145struct Index<'a> {
146    pages: &'a [Page],
147    by_idx: &'a HashMap<usize, usize>,
148    by_id: &'a HashMap<String, usize>,
149}
150
151fn front(c: &Corpus, p: &Page, t: Target, ix: &Index) -> String {
152    let d = &c.docs[p.idx];
153    let fm = d.fm();
154    let get = |k: &str| fm.and_then(|m| m.get(k));
155    let mut lines = vec!["---".to_string()];
156    if t == Target::Docusaurus {
157        let stem = p.file.rsplit('/').next().expect("rsplit yields at least one item").trim_end_matches(".md");
158        lines.push(format!("id: {}", yaml_str(stem)));
159        lines.push(format!("specfold_id: {}", yaml_str(&d.id.to_string())));
160        lines.push(format!("sidebar_label: {}", yaml_str(&p.title)));
161        lines.push(format!("sidebar_position: {}", ix.by_idx[&p.idx] + 1));
162    } else {
163        lines.push(format!("id: {}", yaml_str(&d.id.to_string())));
164    }
165    lines.push(format!("type: {}", yaml_str(&d.id.ty)));
166    if let Some(s) = d.status() {
167        lines.push(format!("status: {}", yaml_str(&s)));
168    }
169    lines.push(format!("title: {}", yaml_str(&p.title)));
170    for k in ["aliases", "tags"] {
171        let v = strings(get(k));
172        if !v.is_empty() {
173            lines.push(format!("{k}: {}", yaml_list(&v)));
174        }
175    }
176    for k in ["created", "updated"] {
177        if let Some(v) = get(k).and_then(|v| v.as_str()) {
178            lines.push(format!("{k}: {}", yaml_str(v)));
179        }
180    }
181    if let Some(Value::Object(rel)) = get("rel") {
182        for (kind, v) in rel {
183            let ids: Vec<String> = strings(Some(v))
184                .into_iter()
185                .map(|s| match (t, g_target(&s, ix)) {
186                    (Target::Obsidian, Some(q)) => format!("[[{}|{}]]", q.file.trim_end_matches(".md"), q.title),
187                    _ => s,
188                })
189                .collect();
190            if !ids.is_empty() {
191                lines.push(format!("rel_{}: {}", kind.replace(['-', '.'], "_"), yaml_list(&ids)));
192            }
193        }
194    }
195    lines.push("---".into());
196    lines.join("\n") + "\n\n"
197}
198
199/// The exported page for a relation target written as an id.
200fn g_target<'a>(id: &str, ix: &Index<'a>) -> Option<&'a Page> {
201    let target = crate::model::RefText::parse(id).target;
202    let idx = ix.by_id.get(target.as_str())?;
203    ix.by_idx.get(idx).map(|&i| &ix.pages[i])
204}
205
206/// GitHub's heading slug: lower case, punctuation dropped, spaces to `-`.
207pub fn github_slug(heading: &str) -> String {
208    heading
209        .trim()
210        .to_lowercase()
211        .chars()
212        .filter_map(|c| {
213            if c.is_alphanumeric() || c == '-' || c == '_' {
214                Some(c)
215            } else if c == ' ' {
216                Some('-')
217            } else {
218                None
219            }
220        })
221        .collect()
222}
223
224/// A relative link from `from` to `to`, both tree paths.
225fn relative(from: &str, to: &str) -> String {
226    let from_dir: Vec<&str> = from.split('/').collect::<Vec<_>>()[..from.matches('/').count()].to_vec();
227    let to_parts: Vec<&str> = to.split('/').collect();
228    let common = from_dir.iter().zip(&to_parts).take_while(|(a, b)| a == b).count();
229    let mut out: Vec<String> = vec!["..".into(); from_dir.len() - common];
230    out.extend(to_parts[common..].iter().map(|s| s.to_string()));
231    out.join("/")
232}
233
234fn rewrite(
235    p: &Page,
236    t: Target,
237    pages: &[Page],
238    by_idx: &HashMap<usize, usize>,
239    unresolved: &mut Vec<Unresolved>,
240    src: &str,
241) -> String {
242    let mut out = String::new();
243    let mut at = 0;
244    for l in &p.links {
245        out.push_str(&p.body[at..l.span.0]);
246        at = l.span.1;
247        let dest = l.idx.and_then(|i| by_idx.get(&i)).map(|&i| &pages[i]);
248        let text = l.label.clone().or_else(|| dest.map(|q| q.title.clone())).or_else(|| l.title.clone());
249        let Some(q) = dest.filter(|_| l.reason.is_none()) else {
250            let reason = l.reason.clone().unwrap_or_else(|| {
251                if l.remote.is_some() {
252                    format!("in remote {}", l.remote.clone().unwrap_or_default())
253                } else {
254                    "not part of this export".into()
255                }
256            });
257            unresolved.push(Unresolved { path: src.into(), raw: l.raw.clone(), reason });
258            let inner = l.raw.trim_start_matches("[[").trim_end_matches("]]");
259            out.push_str(&text.unwrap_or_else(|| inner.split('|').next().unwrap_or(inner).trim().to_string()));
260            continue;
261        };
262        let text = text.unwrap_or_else(|| q.title.clone());
263        let heading = l.anchor.as_ref().map(|a| q.headings.get(a).cloned().unwrap_or_else(|| a.clone()));
264        out.push_str(&match t {
265            Target::Obsidian => {
266                let h = heading.map(|h| format!("#{h}")).unwrap_or_default();
267                format!("[[{}{h}|{text}]]", q.file.trim_end_matches(".md"))
268            }
269            Target::Mkdocs | Target::Docusaurus => {
270                let a = l.anchor.as_ref().map(|a| format!("#{a}")).unwrap_or_default();
271                format!("[{text}]({}{a})", relative(&p.file, &q.file))
272            }
273            Target::GithubWiki => {
274                let a = heading.map(|h| format!("#{}", github_slug(&h))).unwrap_or_default();
275                format!("[{text}]({}{a})", q.file.trim_end_matches(".md"))
276            }
277        });
278    }
279    out.push_str(&p.body[at..]);
280    if t.keeps_anchors() { out } else { strip_anchors(&out) }
281}
282
283/// Removes `{#anchor}` from headings outside code fences.
284fn strip_anchors(text: &str) -> String {
285    let re = regex::Regex::new(r"\s*\{#[A-Za-z0-9][A-Za-z0-9_.-]*\}\s*$").expect("static regex is valid");
286    let mut fence = crate::markdown::Fence::default();
287    let mut out = String::new();
288    for line in text.split_inclusive('\n') {
289        let tr = line.trim_start();
290        fence.step(tr);
291        if !fence.inside() && tr.starts_with('#') {
292            let nl = if line.ends_with('\n') { "\n" } else { "" };
293            out.push_str(&re.replace(line.trim_end_matches(['\n', '\r']), ""));
294            out.push_str(nl);
295        } else {
296            out.push_str(line);
297        }
298    }
299    out
300}
301
302fn index_files(tree: &mut Tree, c: &Corpus, t: Target, pages: &[Page]) {
303    let mut by_type: BTreeMap<&str, Vec<&Page>> = BTreeMap::new();
304    for p in pages {
305        by_type.entry(c.docs[p.idx].id.ty.as_str()).or_default().push(p);
306    }
307    let list = |link: &dyn Fn(&Page) -> String| {
308        let mut s = String::new();
309        for (ty, ps) in &by_type {
310            s.push_str(&format!("\n## {ty}\n\n"));
311            for p in ps {
312                s.push_str(&format!("- {}\n", link(p)));
313            }
314        }
315        s
316    };
317    match t {
318        Target::GithubWiki => {
319            let link = |p: &Page| format!("[{}]({})", p.title, p.file.trim_end_matches(".md"));
320            tree.files.insert("Home.md".into(), format!("# Specs\n{}", list(&link)));
321            tree.files.insert("_Sidebar.md".into(), list(&link).trim_start().to_string());
322        }
323        Target::Mkdocs | Target::Docusaurus => {
324            let link = |p: &Page| format!("[{}]({})", p.title, p.file);
325            tree.files.insert("index.md".into(), format!("# Specs\n{}", list(&link)));
326        }
327        Target::Obsidian => {}
328    }
329}
330
331/// Whether `out` may be written: missing, empty, or holding a previous export's manifest.
332/// Returns the files the previous export wrote.
333/// Writes the tree and its manifest, removing the files of the previous export first.
334pub fn write(out: &Path, tree: &Tree, target: Target, previous: &[String]) -> Result<()> {
335    for f in previous {
336        if f.split('/').any(|s| s == ".." || s.is_empty()) || Path::new(f).is_absolute() {
337            continue;
338        }
339        let p = out.join(f);
340        if p.is_file() {
341            std::fs::remove_file(&p)?;
342            let mut dir = p.parent();
343            while let Some(d) = dir.filter(|d| *d != out) {
344                if std::fs::remove_dir(d).is_err() {
345                    break;
346                }
347                dir = d.parent();
348            }
349        }
350    }
351    for (rel, text) in &tree.files {
352        let p = out.join(rel);
353        std::fs::create_dir_all(p.parent().expect("an output path joined from a file name has a parent"))?;
354        std::fs::write(&p, text)?;
355    }
356    let manifest = json!({
357        "generator": "specfold",
358        "target": target.name(),
359        "files": tree.files.keys().collect::<Vec<_>>(),
360    });
361    // C222: an export of no documents is the manifest alone, in a folder made for it.
362    std::fs::create_dir_all(out)?;
363    std::fs::write(out.join(MANIFEST), serde_json::to_string_pretty(&manifest)? + "\n")?;
364    Ok(())
365}
366
367#[cfg(test)]
368mod tests {
369    use super::*;
370
371    #[test]
372    fn names_and_paths() {
373        assert_eq!(safe_name("a:b/c"), "a-b-c");
374        assert_eq!(relative("shop/adr/a.md", "shop/sot/b.md"), "../sot/b.md");
375        assert_eq!(relative("shop/adr/a.md", "shop/adr/b.md"), "b.md");
376        assert_eq!(github_slug("Cart Totals (v2)"), "cart-totals-v2");
377        assert_eq!(strip_anchors("## Totals {#CO-1}\n```\n## x {#y}\n```\n"), "## Totals\n```\n## x {#y}\n```\n");
378    }
379}
specfold_core/export/markdown/tree.rs

specfold_core/export/markdown/
tree.rs
