oxdock_parser/
markdown.rs

1use anyhow::{Result, bail};
2use line_ending::LineEnding;
3use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
4
5/// Runner configuration parsed from a fenced block's info-string.
6#[derive(Debug, Default, Clone, Eq, PartialEq)]
7pub struct BlockMetadata {
8    pub env: Vec<(String, String)>,
9    pub unified_roots: bool,
10    pub expect_error: Option<String>,
11    /// Targets bound to the in-process mock REMOTE runner
12    /// (```oxdock mock_remote:"prod"`). Lets doc examples execute sealed
13    /// blocks for real with no ssh, no muxio, no tarballs.
14    pub mock_remote: Vec<String>,
15}
16
17/// A fenced code block extracted from a Markdown document.
18#[derive(Debug, Clone, Eq, PartialEq)]
19pub struct FencedBlock {
20    /// 1-based line number of the opening fence in the source document.
21    pub line_no: usize,
22    pub info_string: String,
23    pub metadata: BlockMetadata,
24    pub body: String,
25}
26
27/// Extract fenced code blocks whose info-string starts with `lang`.
28///
29/// Uses `pulldown-cmark` for reliable CommonMark fence parsing. The first
30/// whitespace-delimited token of an opening fence's info-string selects the
31/// language; remaining tokens are scanned as `key:value` metadata.
32pub fn extract_fenced_blocks(markdown: &str, lang: &str) -> Result<Vec<FencedBlock>> {
33    let mut options = Options::empty();
34    options.insert(Options::ENABLE_TABLES);
35    let offset_parser = Parser::new_ext(markdown, options).into_offset_iter();
36
37    // Pre-compute newline byte offsets for line-number mapping.
38    let mut line_starts: Vec<usize> = vec![0];
39    for (i, _) in markdown.match_indices('\n') {
40        line_starts.push(i + 1);
41    }
42
43    let mut blocks = Vec::new();
44    let mut in_block = false;
45    let mut current_line_no = 0;
46    let mut current_info = String::new();
47    let mut current_body = String::new();
48
49    for (event, range) in offset_parser {
50        match event {
51            Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(info))) => {
52                let info_str = info.to_string();
53                let first_token = info_str.split_whitespace().next().unwrap_or("");
54                if first_token == lang {
55                    in_block = true;
56                    // Map byte offset to 1-based line number.
57                    let line_no = line_starts.partition_point(|&start| start <= range.start);
58                    current_line_no = line_no;
59                    current_info = info_str;
60                    current_body.clear();
61                }
62            }
63            Event::Text(text) if in_block => {
64                current_body.push_str(&text);
65            }
66            Event::End(TagEnd::CodeBlock) if in_block => {
67                in_block = false;
68                // pulldown-cmark auto-closes unclosed fences by emitting End
69                // at EOF. Detect this by checking whether the block byte
70                // range terminates with an explicit fence closer.
71                let block_slice = markdown[range.start..range.end].trim_end();
72                if !block_slice.ends_with("```") && !block_slice.ends_with("~~~") {
73                    bail!("markdown:{current_line_no}: code fence is never closed");
74                }
75                let metadata = parse_metadata(&current_info, current_line_no)?;
76                // Trim trailing newline that pulldown-cmark includes from the last line.
77                let body = current_body.trim_end_matches('\n').to_string();
78                blocks.push(FencedBlock {
79                    line_no: current_line_no,
80                    info_string: std::mem::take(&mut current_info),
81                    metadata,
82                    body: LineEnding::normalize(&body),
83                });
84            }
85            _ => {}
86        }
87    }
88
89    if in_block {
90        bail!("markdown:{current_line_no}: code fence is never closed");
91    }
92    Ok(blocks)
93}
94
95/// Extract the `expect_error` needle from a fence info string such as
96/// `oxdock expect_error:"boom"`. Returns `None` when the key is absent.
97/// Malformed metadata errors, exactly like fenced-block extraction, so
98/// generators and runners agree on what a fence means.
99pub fn expect_error_from_info(info: &str) -> Result<Option<String>> {
100    Ok(parse_metadata(info, 0)?.expect_error)
101}
102
103fn skip_token(info: &str) -> &str {
104    let first = info.split_whitespace().next().unwrap_or("");
105    info.trim_start()[first.len()..].trim_start()
106}
107
108fn parse_metadata(info: &str, line_no: usize) -> Result<BlockMetadata> {
109    let fail = |msg: String| anyhow::anyhow!("markdown:{line_no}: {msg}");
110
111    let rest = skip_token(info);
112    let mut metadata = BlockMetadata::default();
113    let mut scanner = rest.chars().peekable();
114
115    loop {
116        while matches!(scanner.peek(), Some(c) if c.is_whitespace()) {
117            scanner.next();
118        }
119        let Some(first_char) = scanner.peek().copied() else {
120            break;
121        };
122        if !first_char.is_ascii_alphanumeric() && first_char != '_' {
123            return Err(fail(format!(
124                "unexpected character '{first_char}' in fence metadata"
125            )));
126        }
127        let mut key = String::new();
128        let mut saw_equals = false;
129        while let Some(&c) = scanner.peek()
130            && c != ':'
131            && c != '='
132            && !c.is_whitespace()
133        {
134            key.push(c);
135            scanner.next();
136        }
137        match scanner.peek() {
138            Some(':') => {
139                scanner.next();
140            }
141            Some('=') => {
142                saw_equals = true;
143            }
144            other => {
145                let _ = other;
146                return Err(fail(format!(
147                    "fence metadata item '{key}' is missing ':' after its key"
148                )));
149            }
150        }
151        if saw_equals {
152            return Err(fail(format!(
153                "fence metadata item '{key}' must use 'key:value' syntax"
154            )));
155        }
156
157        while matches!(scanner.peek(), Some(' ')) {
158            scanner.next();
159        }
160        let value = if scanner.peek() == Some(&'"') {
161            scanner.next();
162            let mut value = String::new();
163            loop {
164                match scanner.next() {
165                    Some('"') => break,
166                    Some(c) => value.push(c),
167                    None => {
168                        return Err(fail(
169                            "unterminated double quote in fence metadata".to_string(),
170                        ));
171                    }
172                }
173            }
174            value
175        } else {
176            let mut value = String::new();
177            while let Some(&c) = scanner.peek()
178                && !c.is_whitespace()
179            {
180                value.push(c);
181                scanner.next();
182            }
183            value
184        };
185
186        match key.as_str() {
187            "env" => {
188                let (name, val) = value
189                    .split_once('=')
190                    .ok_or_else(|| fail("env metadata must be 'env:KEY=VALUE'".to_string()))?;
191                metadata.env.push((name.to_string(), val.to_string()));
192            }
193            "roots" => {
194                if value != "unified" {
195                    return Err(fail(format!(
196                        "unsupported roots metadata '{value}' (only 'unified' is defined)"
197                    )));
198                }
199                if metadata.unified_roots {
200                    return Err(fail("roots metadata specified more than once".to_string()));
201                }
202                metadata.unified_roots = true;
203            }
204            "expect_error" => {
205                if metadata.expect_error.replace(value.clone()).is_some() {
206                    return Err(fail(
207                        "expect_error metadata specified more than once".to_string(),
208                    ));
209                }
210            }
211            "mock_remote" => {
212                metadata.mock_remote.push(value.clone());
213            }
214            other => {
215                return Err(fail(format!(
216                    "unknown fence metadata key '{other}' (supported: env, roots, expect_error, mock_remote)"
217                )));
218            }
219        }
220    }
221    Ok(metadata)
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227    use indoc::indoc;
228
229    #[test]
230    fn extracts_matching_language_blocks_with_line_numbers() {
231        let md = indoc! {r#"
232            before
233
234            ```bash
235            RUN not extracted
236            ```
237
238            ```oxdock
239            ECHO first
240            ```
241
242            ```text oxdock-like
243            ignored because the first token is text
244            ```
245
246            ```oxdock
247            ECHO second
248            ```
249        "#};
250        let blocks = extract_fenced_blocks(md, "oxdock").expect("extract");
251        assert_eq!(blocks.len(), 2);
252        assert_eq!(blocks[0].line_no, 7);
253        assert_eq!(blocks[0].body, "ECHO first");
254        assert_eq!(blocks[1].line_no, 15);
255        assert_eq!(blocks[1].body, "ECHO second");
256        assert_eq!(blocks[0].metadata, BlockMetadata::default());
257    }
258
259    #[test]
260    fn closing_fence_requires_at_least_as_many_ticks() {
261        let md = indoc! {r#"
262            ````oxdock
263            ECHO stays-open-here
264            ``` not a closer
265            ECHO still-body
266            ````
267            ```oxdock
268            ECHO next
269            ```
270        "#};
271        let blocks = extract_fenced_blocks(md, "oxdock").expect("extract");
272        assert_eq!(blocks.len(), 2);
273        assert_eq!(
274            blocks[0].body,
275            "ECHO stays-open-here\n``` not a closer\nECHO still-body"
276        );
277    }
278
279    #[test]
280    fn unclosed_fence_is_an_error() {
281        let err =
282            extract_fenced_blocks("```oxdock\nECHO x\n", "oxdock").expect_err("unclosed fence");
283        assert!(err.to_string().contains("never closed"));
284    }
285
286    #[test]
287    fn expect_error_needle_reads_from_info_string() {
288        assert_eq!(
289            expect_error_from_info("oxdock expect_error:\"undefined variable\"")
290                .expect("parse")
291                .as_deref(),
292            Some("undefined variable")
293        );
294        assert_eq!(
295            expect_error_from_info("oxdock roots:unified")
296                .expect("parse")
297                .as_deref(),
298            None
299        );
300        assert!(expect_error_from_info("oxdock expect_error").is_err());
301    }
302
303    #[test]
304    fn parses_quoted_values_containing_spaces() {
305        let md = "```oxdock expect_error:\"EXIT requested with code 42\"\nEXIT 42\n```\n";
306        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
307        assert_eq!(
308            block.metadata.expect_error.as_deref(),
309            Some("EXIT requested with code 42")
310        );
311    }
312
313    #[test]
314    fn parses_env_and_roots_metadata() {
315        let md = concat!(
316            "```oxdock env:A=1 env:B=two roots:unified\n",
317            "ECHO x\n",
318            "```\n"
319        );
320        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
321        assert_eq!(
322            block.metadata.env,
323            vec![
324                ("A".to_string(), "1".to_string()),
325                ("B".to_string(), "two".to_string())
326            ]
327        );
328        assert!(block.metadata.unified_roots);
329    }
330
331    #[test]
332    fn unknown_and_duplicate_keys_are_errors() {
333        let md = "```oxdock bogus:x\nECHO x\n```\n";
334        let err = extract_fenced_blocks(md, "oxdock").expect_err("unknown key");
335        assert!(
336            err.to_string()
337                .contains("unknown fence metadata key 'bogus'")
338        );
339
340        let roots = "```oxdock roots:squashed\nECHO x\n```\n";
341        let err = extract_fenced_blocks(roots, "oxdock").expect_err("bad roots");
342        assert!(err.to_string().contains("only 'unified'"));
343
344        let env = "```oxdock env:NOVALUE\nECHO x\n```\n";
345        let err = extract_fenced_blocks(env, "oxdock").expect_err("env without '='");
346        assert!(err.to_string().contains("env:KEY=VALUE"));
347    }
348
349    #[test]
350    fn unterminated_quote_is_an_error() {
351        let md = "```oxdock expect_error:\"oops\nECHO x\n```\n";
352        let err = extract_fenced_blocks(md, "oxdock").expect_err("unterminated quote");
353        assert!(err.to_string().contains("unterminated double quote"));
354    }
355
356    #[test]
357    fn equals_separator_is_rejected_with_guidance() {
358        let md = "```oxdock expect_error=\"oops\"\nECHO x\n```\n"
359            .replace("expect_error:", "expect_error=");
360        let err = extract_fenced_blocks(&md, "oxdock").expect_err("equals separator");
361        assert!(err.to_string().contains("must use 'key:value' syntax"));
362    }
363
364    #[test]
365    fn crlf_bodies_are_normalized_to_lf() {
366        let md = "```oxdock\r\nECHO a\r\nECHO b\r\n```\r\n";
367        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
368        assert_eq!(block.body, "ECHO a\nECHO b");
369    }
370
371    #[test]
372    fn bare_cr_bodies_are_normalized_to_lf() {
373        let md = "```oxdock\nECHO a\rECHO b\r\n```\n";
374        let block = &extract_fenced_blocks(md, "oxdock").expect("extract")[0];
375        assert_eq!(block.body, "ECHO a\nECHO b");
376    }
377}
oxdock_parser/markdown.rs

oxdock_parser/
markdown.rs
