marmite 0.4.2

Marmite is the easiest static site generator.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
use crate::config::ParserOptions;
use crate::highlight::MarmiteHighlighter;
use crate::re;
use crate::site::Data;
use comrak::{
    markdown_to_html, markdown_to_html_with_plugins, options::BrokenLinkReference,
    options::Plugins, options::RenderPlugins, Options, ResolvedReference,
};
use frontmatter_gen::{detect_format, extract_raw_frontmatter, parse, Frontmatter};
use log::warn;
use regex::Regex;
use std::fmt::Write as _;
use std::io::Write as IoWrite;

use std::fs;
use std::path::Path;
use std::sync::Arc;
use url::Url;
use urlencoding::decode as urldecode;

const MEDIA_EXTENSIONS: &[&str] = &[
    ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".avif", ".bmp", ".tiff", ".tif", ".ico",
    ".pdf", ".mp4", ".mov", ".avi", ".mkv", ".webm", ".mp3", ".wav", ".ogg", ".flac", ".zip",
    ".tar", ".gz", ".7z", ".rar", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".txt",
    ".csv", ".json", ".xml", ".yaml", ".yml", ".toml",
];

pub fn append_references(content: &str, references_path: &Path) -> String {
    if references_path.exists() {
        let references = fs::read_to_string(references_path).unwrap_or_default();
        format!("{content}\n\n{references}")
    } else {
        content.to_string()
    }
}

/// Extract all the internal links from the html content
/// that point to a internal .html file (excluding http links)
/// and return them as a vector of strings
pub fn get_links_to(html: &str) -> Option<Vec<String>> {
    let mut result = Vec::new();
    let re = Regex::new(re::CAPTURE_SLUG_ANCHOR_FROM_HREF).expect("Links regex should compile");
    for cap in re.captures_iter(html) {
        if let Some(m) = cap.get(1) {
            let href = m.as_str();
            if !href.starts_with("http") {
                let page = href.trim_start_matches("./").to_string();
                let heading = cap.get(2).map_or("", |h| h.as_str());
                result.push(format!("{page}{heading}").to_string());
            }
        }
    }
    if result.is_empty() {
        return None;
    }
    Some(result)
}

pub fn get_media_links_to(html: &str) -> Option<Vec<String>> {
    let mut result = Vec::new();
    let re = Regex::new(re::CAPTURE_MEDIA_SRC_OR_HREF).expect("Media links regex should compile");
    for cap in re.captures_iter(html) {
        if let Some(m) = cap.get(1) {
            let value = m.as_str();
            if value.starts_with("http") || value.starts_with("//") || value.starts_with('#') {
                continue;
            }
            let lower = value.to_lowercase();
            if MEDIA_EXTENSIONS.iter().any(|ext| lower.ends_with(ext)) {
                let normalized = value.trim_start_matches("./").to_string();
                if !result.contains(&normalized) {
                    result.push(normalized);
                }
            }
        }
    }
    if result.is_empty() {
        return None;
    }
    Some(result)
}

#[allow(clippy::needless_pass_by_value)]
fn warn_broken_link(link_ref: BrokenLinkReference) -> Option<ResolvedReference> {
    let original = link_ref.original;
    let is_allowed = original
        .starts_with("http")  // external links
        || original.starts_with('!') // Callouts
        || original.starts_with('#') // anchors
        ||original.starts_with('^') // footnotes
        || original.starts_with('/') // absolute links
        || (original.len() == 1 && original.chars().next().is_some_and(|c| !c.is_ascii_digit())) // task checkboxes
        || original.is_empty(); // empty links
    if !is_allowed {
        warn!("Reference missing: [{original}] - add '[{original}]: url' to the end of your content file or to the '_references.md' file.");
    }
    None
}

pub fn get_table_of_contents_from_html(html: &str) -> String {
    let re = Regex::new(re::CAPTURE_LEVEL_ID_CONTENT_FROM_H_TAG)
        .expect("Table of contents regex should compile");
    let anchor_re =
        Regex::new(re::MATCH_COMRAK_HEADING_ANCHOR).expect("Anchor strip regex should compile");
    let mut toc = String::new();
    let mut last_level = 0;

    for cap in re.captures_iter(html) {
        let level = cap.get(1).map_or(0, |m| m.as_str().parse().unwrap_or(0));
        let raw_content = cap.get(3).map_or("", |m| m.as_str());
        let title = anchor_re.replace_all(raw_content, "");
        let title = title.trim();
        let slug = cap.get(2).map_or_else(
            || format!("#{}", crate::slugify::slugify(title)),
            |m| format!("#{}", m.as_str()),
        );

        match level.cmp(&last_level) {
            std::cmp::Ordering::Greater => {
                for _ in last_level..level {
                    toc.push_str("<ul>\n");
                }
            }
            std::cmp::Ordering::Less => {
                for _ in level..last_level {
                    toc.push_str("</ul>\n");
                }
            }
            std::cmp::Ordering::Equal => {}
        }

        let _ = writeln!(toc, "<li><a href=\"{slug}\">{title}</a></li>");
        last_level = level;
    }

    for _ in 0..last_level {
        toc.push_str("</ul>\n");
    }

    toc
}

/// Convert markdown to html using comrak with configurable options
pub fn get_html_with_options(
    markdown: &str,
    parser_options: &ParserOptions,
    highlighter: Option<&MarmiteHighlighter>,
) -> String {
    let mut options = Options::default();

    // Apply configurable render options
    options.render.figure_with_caption = parser_options.render.figure_with_caption;
    options.render.ignore_empty_links = parser_options.render.ignore_empty_links;
    options.render.r#unsafe = parser_options.render.unsafe_;
    options.render.sourcepos = parser_options.render.sourcepos;

    // Apply configurable parse options
    options.parse.broken_link_callback = Some(Arc::new(warn_broken_link)); // Not configurable
    options.parse.relaxed_tasklist_matching = parser_options.parse.relaxed_tasklist_matching;

    // Apply configurable extension options
    options.extension.alerts = parser_options.extension.alerts;
    options.extension.autolink = parser_options.extension.autolink;
    options.extension.description_lists = parser_options.extension.description_lists;
    options.extension.footnotes = parser_options.extension.footnotes;
    options.extension.greentext = parser_options.extension.greentext;
    options.extension.header_id_prefix = Some(String::new());
    // options.extension.image_url_rewriter = TODO: implement this to point to a resized image
    options.extension.multiline_block_quotes = parser_options.extension.multiline_block_quotes;
    options.extension.tagfilter = parser_options.extension.tagfilter;
    options.extension.shortcodes = parser_options.extension.shortcodes;
    options.extension.spoiler = parser_options.extension.spoiler;
    options.extension.strikethrough = parser_options.extension.strikethrough;
    options.extension.table = parser_options.extension.table;
    options.extension.tasklist = parser_options.extension.tasklist;
    options.extension.underline = parser_options.extension.underline;
    options.extension.wikilinks_title_before_pipe =
        parser_options.extension.wikilinks_title_before_pipe;
    options.extension.wikilinks_title_after_pipe =
        parser_options.extension.wikilinks_title_after_pipe;

    // Apply syntax highlighting (or not)
    let rendered = if let Some(hl) = highlighter {
        let render_plugins = RenderPlugins {
            codefence_syntax_highlighter: Some(hl),
            ..RenderPlugins::default()
        };
        let plugins = Plugins {
            render: render_plugins,
        };
        markdown_to_html_with_plugins(markdown, &options, &plugins)
    } else {
        markdown_to_html(markdown, &options)
    };
    fix_internal_links(&rendered)
}

/// Takes the html content, finds all the internal links and
/// fixes them to point to the correct html file
/// Also removes the .md|.html extension from the text of the link
pub fn fix_internal_links(html: &str) -> String {
    let re = Regex::new(re::CAPTURE_LINK_AND_TEXT_FROM_A_TAG)
        .expect("Fix internal links regex should compile");
    re.replace_all(html, |caps: &regex::Captures| {
        let link = caps.get(0).map_or("", |m| m.as_str());
        let href = caps.get(1).map_or("", |m| m.as_str());
        let text = caps.get(2).map_or("", |m| m.as_str());
        let href_lower = href.to_lowercase();
        let is_media_file = MEDIA_EXTENSIONS.iter().any(|ext| href_lower.ends_with(ext));

        if link.contains("class=\"anchor\"")
            || link.contains("data-footnote-ref")
            || link.contains("footnote-backref")
            || link.starts_with('/')
            || href.starts_with('.')
            || is_media_file
        {
            return link.to_string();
        }

        if let Ok(url) = Url::parse(href) {
            if !url.scheme().is_empty() {
                return link.to_string();
            }
        }

        let new_href = if let Ok(parsed) = Url::parse(&format!("m://m/{href}")) {
            let raw_path = parsed
                .path()
                .trim_start_matches('/')
                .trim_end_matches(".md")
                .trim_end_matches(".html");
            let decoded_path = urldecode(raw_path).unwrap_or_else(|_| raw_path.into());
            let path = crate::slugify::slugify(&*decoded_path);
            let fragment = match parsed.fragment() {
                Some(f) => {
                    let decoded_f = urldecode(f).unwrap_or_else(|_| f.into());
                    crate::slugify::slugify(&*decoded_f)
                }
                None => String::new(),
            };

            let mut new_href = String::new();
            if !path.is_empty() {
                let _ = write!(new_href, "{path}.html");
            }
            if !fragment.is_empty() {
                let _ = write!(new_href, "#{fragment}");
            }
            new_href
        } else {
            href.to_string()
        };

        let new_text = text
            .trim_start_matches('#')
            .trim_end_matches(".md")
            .trim_end_matches(".html")
            .replace('#', " > ");

        link.replace(&format!("href=\"{href}\""), &format!("href=\"{new_href}\""))
            .replace(&format!(">{text}</a>"), &format!(">{new_text}</a>"))
    })
    .to_string()
}

/// Decode common HTML entities in text
/// Handles &amp;, &lt;, &gt;, &quot;, &#39; and numeric entities
fn decode_html_entities(text: &str) -> String {
    text.replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#39;", "'")
        .replace("&#x27;", "'")
}

/// Convert a `serde_yaml::Value` mermaid config into `RenderOptions`.
///
/// Serializes the YAML value to JSON, writes it to a temp file, and calls
/// `mermaid_rs_renderer::load_config()` which handles all the camelCase
/// field mapping (themeVariables, flowchart, etc.) internally.
fn yaml_config_to_render_options(
    config: &serde_yaml::Value,
    slug: &str,
) -> Result<mermaid_rs_renderer::RenderOptions, String> {
    let json_string = serde_json::to_string(config)
        .map_err(|e| format!("Failed to serialize mermaid_config to JSON: {e}"))?;

    let mut tmp = tempfile::NamedTempFile::new()
        .map_err(|e| format!("Failed to create temp file for mermaid config: {e}"))?;
    tmp.write_all(json_string.as_bytes())
        .map_err(|e| format!("Failed to write mermaid config to temp file: {e}"))?;
    tmp.flush()
        .map_err(|e| format!("Failed to flush temp file: {e}"))?;

    let mermaid_config = mermaid_rs_renderer::config::load_config(Some(tmp.path()))
        .map_err(|e| format!("Failed to parse mermaid_config for '{slug}': {e}"))?;

    Ok(mermaid_rs_renderer::RenderOptions {
        theme: mermaid_config.theme,
        layout: mermaid_config.layout,
    })
}

/// Replace mermaid code blocks in HTML with SVG rendered at build time.
pub fn render_native_mermaid(
    html: &str,
    slug: &str,
    mermaid_config: Option<&serde_yaml::Value>,
) -> String {
    let re = Regex::new(re::CAPTURE_MERMAID_BLOCK).expect("Mermaid block regex should compile");

    let render_options =
        mermaid_config.and_then(|cfg| match yaml_config_to_render_options(cfg, slug) {
            Ok(opts) => Some(opts),
            Err(e) => {
                warn!("Invalid mermaid_config for '{slug}': {e}; using defaults");
                None
            }
        });

    re.replace_all(html, |caps: &regex::Captures| {
        let full_match = caps.get(0).map_or("", |m| m.as_str());
        let escaped_source = caps.get(1).map_or("", |m| m.as_str());
        let source = decode_html_entities(escaped_source);

        let render_result = if let Some(ref opts) = render_options {
            mermaid_rs_renderer::render_with_options(&source, opts.clone())
        } else {
            mermaid_rs_renderer::render(&source)
        };

        match render_result {
            Ok(svg) => {
                format!("<div class=\"mermaid-diagram\">{svg}</div>")
            }
            Err(e) => {
                warn!("Failed to render mermaid diagram in '{slug}': {e}; keeping raw source");
                full_match.to_string()
            }
        }
    })
    .to_string()
}

/// Find content by title in site data (case-insensitive)
/// Returns the slug of the matching content if found
fn find_content_by_title(title: &str, site_data: &Data) -> Option<String> {
    let title_lower = title.to_lowercase();

    // Search in posts first
    for content in &site_data.posts {
        if content.title.to_lowercase() == title_lower {
            return Some(format!("{}.html", content.slug));
        }
    }

    // Search in pages
    for content in &site_data.pages {
        if content.title.to_lowercase() == title_lower {
            return Some(format!("{}.html", content.slug));
        }
    }

    None
}

/// Fix wikilinks in HTML by replacing filename-based hrefs with slug-based hrefs
/// This function processes HTML that contains wikilinks with data-wikilink="true"
/// and attempts to match the link titles with actual content titles to use proper slugs
pub fn fix_wikilinks(html: &str, site_data: &Data) -> String {
    let re =
        Regex::new(re::CAPTURE_WIKILINK_HREF_AND_TITLE).expect("Wikilink regex should compile");

    re.replace_all(html, |caps: &regex::Captures| {
        let original_link = caps.get(0).map_or("", |m| m.as_str());
        let original_href = caps.get(1).map_or("", |m| m.as_str());
        let link_title = caps.get(2).map_or("", |m| m.as_str());

        // Decode HTML entities from the link title
        let decoded_title = decode_html_entities(link_title);

        // Try to find matching content by title
        if let Some(proper_href) = find_content_by_title(&decoded_title, site_data) {
            // Replace the href with the proper slug-based href
            original_link.replace(
                &format!("href=\"{original_href}\""),
                &format!("href=\"{proper_href}\""),
            )
        } else {
            // No match found, keep original
            original_link.to_string()
        }
    })
    .to_string()
}

pub fn parse_front_matter(content: &str) -> Result<(Frontmatter, &str), String> {
    let content = content.trim_start_matches('\n');
    let has_frontmatter =
        content.starts_with("---") || content.starts_with("+++") || content.starts_with('{');
    if !has_frontmatter {
        return Ok((Frontmatter::new(), content));
    }
    extract_fm_content(content)
}

pub fn extract_fm_content(content: &str) -> Result<(Frontmatter, &str), String> {
    let (raw_frontmatter, remaining_content) = extract_raw_frontmatter(content)?;
    let format = detect_format(raw_frontmatter)?;
    let frontmatter = parse(raw_frontmatter, format)?;
    Ok((frontmatter, remaining_content))
}

#[cfg(test)]
#[path = "tests/parser.rs"]
mod tests;