oxdock_parser/
markdown.rs

1use anyhow::{Result, bail};
2use line_ending::LineEnding;
3use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
4
5/// Runner configuration parsed from a fenced block's info-string.
6#[derive(Debug, Default, Clone, Eq, PartialEq)]
7pub struct BlockMetadata {
8    pub env: Vec<(String, String)>,
9    pub unified_roots: bool,
10    pub expect_error: Option<String>,
11}
12
13/// A fenced code block extracted from a Markdown document.
14#[derive(Debug, Clone, Eq, PartialEq)]
15pub struct FencedBlock {
16    /// 1-based line number of the opening fence in the source document.
17    pub line_no: usize,
18    pub info_string: String,
19    pub metadata: BlockMetadata,
20    pub body: String,
21}
22
23/// Extract fenced code blocks whose info-string starts with `lang`.
24///
25/// Uses `pulldown-cmark` for reliable CommonMark fence parsing. The first
26/// whitespace-delimited token of an opening fence's info-string selects the
27/// language; remaining tokens are scanned as `key:value` metadata.
28pub fn extract_fenced_blocks(markdown: &str, lang: &str) -> Result<Vec<FencedBlock>> {
29    let mut options = Options::empty();
30    options.insert(Options::ENABLE_TABLES);
31    let offset_parser = Parser::new_ext(markdown, options).into_offset_iter();
32
33    // Pre-compute newline byte offsets for line-number mapping.
34    let mut line_starts: Vec<usize> = vec![0];
35    for (i, _) in markdown.match_indices('\n') {
36        line_starts.push(i + 1);
37    }
38
39    let mut blocks = Vec::new();
40    let mut in_block = false;
41    let mut current_line_no = 0;
42    let mut current_info = String::new();
43    let mut current_body = String::new();
44
45    for (event, range) in offset_parser {
46        match event {
47            Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(info))) => {
48                let info_str = info.to_string();
49                let first_token = info_str.split_whitespace().next().unwrap_or("");
50                if first_token == lang {
51                    in_block = true;
52                    // Map byte offset to 1-based line number.
53                    let line_no = line_starts.partition_point(|&start| start <= range.start);
54                    current_line_no = line_no;
55                    current_info = info_str;
56                    current_body.clear();
57                }
58            }
59            Event::Text(text) if in_block => {
60                current_body.push_str(&text);
61            }
62            Event::End(TagEnd::CodeBlock) if in_block => {
63                in_block = false;
64                // pulldown-cmark auto-closes unclosed fences by emitting End
65                // at EOF. Detect this by checking whether the block byte
66                // range terminates with an explicit fence closer.
67                let block_slice = markdown[range.start..range.end].trim_end();
68                if !block_slice.ends_with("```") && !block_slice.ends_with("~~~") {
69                    bail!("markdown:{current_line_no}: code fence is never closed");
70                }
71                let metadata = parse_metadata(&current_info, current_line_no)?;
72                // Trim trailing newline that pulldown-cmark includes from the last line.
73                let body = current_body.trim_end_matches('\n').to_string();
74                blocks.push(FencedBlock {
75                    line_no: current_line_no,
76                    info_string: std::mem::take(&mut current_info),
77                    metadata,
78                    body: LineEnding::normalize(&body),
79                });
80            }
81            _ => {}
82        }
83    }
84
85    if in_block {
86        bail!("markdown:{current_line_no}: code fence is never closed");
87    }
88    Ok(blocks)
89}
90
91/// Extract the `expect_error` needle from a fence info string such as
92/// `oxdock expect_error:"boom"`. Returns `None` when the key is absent.
93/// Malformed metadata errors, exactly like fenced-block extraction, so
94/// generators and runners agree on what a fence means.
95pub fn expect_error_from_info(info: &str) -> Result<Option<String>> {
96    Ok(parse_metadata(info, 0)?.expect_error)
97}
98
99fn skip_token(info: &str) -> &str {
100    let first = info.split_whitespace().next().unwrap_or("");
101    info.trim_start()[first.len()..].trim_start()
102}
103
104fn parse_metadata(info: &str, line_no: usize) -> Result<BlockMetadata> {
105    let fail = |msg: String| anyhow::anyhow!("markdown:{line_no}: {msg}");
106
107    let rest = skip_token(info);
108    let mut metadata = BlockMetadata::default();
109    let mut scanner = rest.chars().peekable();
110
111    loop {
112        while matches!(scanner.peek(), Some(c) if c.is_whitespace()) {
113            scanner.next();
114        }
115        let Some(first_char) = scanner.peek().copied() else {
116            break;
117        };
118        if !first_char.is_ascii_alphanumeric() && first_char != '_' {
119            return Err(fail(format!(
120                "unexpected character '{first_char}' in fence metadata"
121            )));
122        }
123        let mut key = String::new();
124        let mut saw_equals = false;
125        while let Some(&c) = scanner.peek()
126            && c != ':'
127            && c != '='
128            && !c.is_whitespace()
129        {
130            key.push(c);
131            scanner.next();
132        }
133        match scanner.peek() {
134            Some(':') => {
135                scanner.next();
136            }
137            Some('=') => {
138                saw_equals = true;
139            }
140            other => {
141                let _ = other;
142                return Err(fail(format!(
143                    "fence metadata item '{key}' is missing ':' after its key"
144                )));
145            }
146        }
147        if saw_equals {
148            return Err(fail(format!(
149                "fence metadata item '{key}' must use 'key:value' syntax"
150            )));
151        }
152
153        while matches!(scanner.peek(), Some(' ')) {
154            scanner.next();
155        }
156        let value = if scanner.peek() == Some(&'"') {
157            scanner.next();
158            let mut value = String::new();
159            loop {
160                match scanner.next() {
161                    Some('"') => break,
162                    Some(c) => value.push(c),
163                    None => {
164                        return Err(fail(
165                            "unterminated double quote in fence metadata".to_string(),
166                        ));
167                    }
168                }
169            }
170            value
171        } else {
172            let mut value = String::new();
173            while let Some(&c) = scanner.peek()
174                && !c.is_whitespace()
175            {
176                value.push(c);
177                scanner.next();
178            }
179            value
180        };
181
182        match key.as_str() {
183            "env" => {
184                let (name, val) = value
185                    .split_once('=')
186                    .ok_or_else(|| fail("env metadata must be 'env:KEY=VALUE'".to_string()))?;
187                metadata.env.push((name.to_string(), val.to_string()));
188            }
189            "roots" => {
190                if value != "unified" {
191                    return Err(fail(format!(
192                        "unsupported roots metadata '{value}' (only 'unified' is defined)"
193                    )));
194                }
195                if metadata.unified_roots {
196                    return Err(fail("roots metadata specified more than once".to_string()));
197                }
198                metadata.unified_roots = true;
199            }
200            "expect_error" => {
201                if metadata.expect_error.replace(value.clone()).is_some() {
202                    return Err(fail(
203                        "expect_error metadata specified more than once".to_string(),
204                    ));
205                }
206            }
207            other => {
208                return Err(fail(format!(
209                    "unknown fence metadata key '{other}' (supported: env, roots, expect_error)"
210                )));
211            }
212        }
213    }
214    Ok(metadata)
215}
216
217#[cfg(test)]
218mod tests {
219    use super::*;
220    use indoc::indoc;
221
222    #[test]
223    fn extracts_matching_language_blocks_with_line_numbers() {
224        let md = indoc! {r#"
225            before
226
227            ```bash
228            RUN not extracted
229            ```
230
231            ```oxdock
232            ECHO first
233            ```
234
235            ```text oxdock-like
236            ignored because the first token is text
237            ```
238
239            ```oxdock
240            ECHO second
241            ```
242        "#};
243        let blocks = extract_fenced_blocks(md, "oxdock").expect("extract");
244        assert_eq!(blocks.len(), 2);
245        assert_eq!(blocks[0].line_no, 7);
246        assert_eq!(blocks[0].body, "ECHO first");
247        assert_eq!(blocks[1].line_no, 15);
248        assert_eq!(blocks[1].body, "ECHO second");
249        assert_eq!(blocks[0].metadata, BlockMetadata::default());
250    }
251
252    #[test]
253    fn closing_fence_requires_at_least_as_many_ticks() {
254        let md = indoc! {r#"
255            ````oxdock
256            ECHO stays-open-here
257            ``` not a closer
258            ECHO still-body
259            ````
260            ```oxdock
261            ECHO next
262            ```
263        "#};
264        let blocks = extract_fenced_blocks(md, "oxdock").expect("extract");
265        assert_eq!(blocks.len(), 2);
266        assert_eq!(
267            blocks[0].body,
268            "ECHO stays-open-here\n``` not a closer\nECHO still-body"
269        );
270    }
271
272    #[test]
273    fn unclosed_fence_is_an_error() {
274        let err =
275            extract_fenced_blocks("```oxdock\nECHO x\n", "oxdock").expect_err("unclosed fence");
276        assert!(err.to_string().contains("never closed"));
277    }
278
279    #[test]
280    fn expect_error_needle_reads_from_info_string() {
281        assert_eq!(
282            expect_error_from_info("oxdock expect_error:\"undefined variable\"")
283                .expect("parse")
284                .as_deref(),
285            Some("undefined variable")
286        );
287        assert_eq!(
288            expect_error_from_info("oxdock roots:unified")
289                .expect("parse")
290                .as_deref(),
291            None
292        );
293        assert!(expect_error_from_info("oxdock expect_error").is_err());
294    }
295
296    #[test]
297    fn parses_quoted_values_containing_spaces() {
298        let md = "```oxdock expect_error:\"EXIT requested with code 42\"\nEXIT 42\n```\n";
299        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
300        assert_eq!(
301            block.metadata.expect_error.as_deref(),
302            Some("EXIT requested with code 42")
303        );
304    }
305
306    #[test]
307    fn parses_env_and_roots_metadata() {
308        let md = concat!(
309            "```oxdock env:A=1 env:B=two roots:unified\n",
310            "ECHO x\n",
311            "```\n"
312        );
313        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
314        assert_eq!(
315            block.metadata.env,
316            vec![
317                ("A".to_string(), "1".to_string()),
318                ("B".to_string(), "two".to_string())
319            ]
320        );
321        assert!(block.metadata.unified_roots);
322    }
323
324    #[test]
325    fn unknown_and_duplicate_keys_are_errors() {
326        let md = "```oxdock bogus:x\nECHO x\n```\n";
327        let err = extract_fenced_blocks(md, "oxdock").expect_err("unknown key");
328        assert!(
329            err.to_string()
330                .contains("unknown fence metadata key 'bogus'")
331        );
332
333        let roots = "```oxdock roots:squashed\nECHO x\n```\n";
334        let err = extract_fenced_blocks(roots, "oxdock").expect_err("bad roots");
335        assert!(err.to_string().contains("only 'unified'"));
336
337        let env = "```oxdock env:NOVALUE\nECHO x\n```\n";
338        let err = extract_fenced_blocks(env, "oxdock").expect_err("env without '='");
339        assert!(err.to_string().contains("env:KEY=VALUE"));
340    }
341
342    #[test]
343    fn unterminated_quote_is_an_error() {
344        let md = "```oxdock expect_error:\"oops\nECHO x\n```\n";
345        let err = extract_fenced_blocks(md, "oxdock").expect_err("unterminated quote");
346        assert!(err.to_string().contains("unterminated double quote"));
347    }
348
349    #[test]
350    fn equals_separator_is_rejected_with_guidance() {
351        let md = "```oxdock expect_error=\"oops\"\nECHO x\n```\n"
352            .replace("expect_error:", "expect_error=");
353        let err = extract_fenced_blocks(&md, "oxdock").expect_err("equals separator");
354        assert!(err.to_string().contains("must use 'key:value' syntax"));
355    }
356
357    #[test]
358    fn crlf_bodies_are_normalized_to_lf() {
359        let md = "```oxdock\r\nECHO a\r\nECHO b\r\n```\r\n";
360        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
361        assert_eq!(block.body, "ECHO a\nECHO b");
362    }
363
364    #[test]
365    fn bare_cr_bodies_are_normalized_to_lf() {
366        let md = "```oxdock\nECHO a\rECHO b\r\n```\n";
367        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
368        assert_eq!(block.body, "ECHO a\nECHO b");
369    }
370}
oxdock_parser/markdown.rs

oxdock_parser/
markdown.rs
