Skip to main content

cpd_tokenizer/
markdown.rs

1// markdown.rs
2
3use std::collections::BTreeMap;
4
5use cpd_core::models::{DetectionToken, Location, Token, TokenKind};
6
7use crate::embedded::blank_ranges_preserve_newlines;
8use crate::formats::resolve_format;
9use crate::line_index::LineIndex;
10use crate::tokenizer::{Mode, TokenMap, TokenizeOptions, push_token, tokenize_format_to_detection};
11
12pub struct LineSpan {
13    pub start: usize,
14    pub end: usize,
15    pub next_start: usize,
16}
17
18pub fn line_spans(content: &str) -> Vec<LineSpan> {
19    if content.is_empty() {
20        return Vec::new();
21    }
22    let mut spans = Vec::new();
23    let bytes = content.as_bytes();
24    let len = bytes.len();
25    let mut pos = 0usize;
26    while pos <= len {
27        let line_end = bytes[pos..]
28            .iter()
29            .position(|&b| b == b'\n')
30            .map(|i| pos + i)
31            .unwrap_or(len);
32        let content_end = if line_end > pos && bytes[line_end - 1] == b'\r' {
33            line_end - 1
34        } else {
35            line_end
36        };
37        let next_start = if line_end < len { line_end + 1 } else { len };
38        spans.push(LineSpan {
39            start: pos,
40            end: content_end,
41            next_start,
42        });
43        if next_start <= pos {
44            break;
45        }
46        pos = next_start;
47        if pos >= len && (len == 0 || bytes[len - 1] != b'\n') {
48            break;
49        }
50    }
51    spans
52}
53
54#[derive(Debug, Clone)]
55struct MarkdownFence {
56    format: String,
57    block_start: usize,
58    inner_start: usize,
59    inner_end: usize,
60    block_end: usize,
61}
62
63struct FenceOpen {
64    marker: u8,
65    len: usize,
66    info: String,
67}
68
69fn parse_opening_fence(line: &str) -> Option<FenceOpen> {
70    // ponytail: split on first byte so the marker check and run-length are separate phases
71    let (&first, rest) = line.as_bytes().split_first()?;
72    let marker = first;
73    if !matches!(marker, b'`' | b'~') {
74        return None;
75    }
76    let len = 1 + rest.iter().take_while(|&&b| b == marker).count();
77    if len < 3 {
78        return None;
79    }
80    Some(FenceOpen {
81        marker,
82        len,
83        info: line[len..].trim().to_string(),
84    })
85}
86
87fn is_closing_fence(line: &str, open: &FenceOpen) -> bool {
88    let trimmed = line.trim();
89    let bytes = trimmed.as_bytes();
90    if bytes.is_empty() {
91        return false;
92    }
93    let len = bytes.iter().take_while(|&&b| b == open.marker).count();
94    len >= open.len && bytes[len..].iter().all(|&b| b == b' ' || b == b'\t')
95}
96
97fn resolve_fence_format(info: &str) -> Option<&'static str> {
98    let tag = info.split_whitespace().next()?;
99    resolve_format(tag)
100}
101
102fn fence_bounds(
103    content: &str,
104    lines: &[LineSpan],
105    close_idx: usize,
106    inner_start: usize,
107) -> (usize, usize) {
108    let inner_end = content[..lines[close_idx].start]
109        .strip_suffix('\n')
110        .map(|prefix| prefix.len())
111        .unwrap_or(lines[close_idx].start)
112        .max(inner_start);
113    let block_end = lines[close_idx].next_start.min(content.len());
114    (inner_end, block_end)
115}
116
117fn extract_code_fences(content: &str) -> Vec<MarkdownFence> {
118    let lines = line_spans(content);
119    let mut fences = Vec::new();
120    let mut idx = 0usize;
121    while idx < lines.len() {
122        let line_text = &content[lines[idx].start..lines[idx].end];
123        let Some(open) = parse_opening_fence(line_text) else {
124            idx += 1;
125            continue;
126        };
127        let resolved = resolve_fence_format(&open.info);
128        let close_idx = lines[idx + 1..]
129            .iter()
130            .position(|span| {
131                let candidate = &content[span.start..span.end];
132                is_closing_fence(candidate, &open)
133            })
134            .map(|p| idx + 1 + p);
135        let Some(close_idx) = close_idx else {
136            idx += 1;
137            continue;
138        };
139        let inner_start = lines
140            .get(idx + 1)
141            .map(|s| s.start)
142            .unwrap_or(lines[idx].next_start);
143        let (inner_end, block_end) = fence_bounds(content, &lines, close_idx, inner_start);
144        let format = resolved.map(|r| r.to_string()).unwrap_or_else(|| {
145            open.info
146                .split_whitespace()
147                .next()
148                .unwrap_or("")
149                .to_string()
150        });
151        fences.push(MarkdownFence {
152            format,
153            block_start: lines[idx].start,
154            inner_start,
155            inner_end,
156            block_end,
157        });
158        idx = close_idx + 1;
159    }
160    fences
161}
162
163fn extract_front_matter(content: &str) -> Option<MarkdownFence> {
164    // ponytail: front-matter opener is `---` on its own line; reject anything else
165    // up front so the line scan only runs on real candidates.
166    let opener_len = if content.starts_with("---\r\n") {
167        5
168    } else if content.starts_with("---\n") {
169        4
170    } else {
171        return None;
172    };
173
174    let lines = line_spans(content);
175    // Body begins right after the opener line.
176    let inner_start = opener_len;
177    let close_idx = front_matter_closer(content, &lines)?;
178    let (inner_end, block_end) = fence_bounds(content, &lines, close_idx, inner_start);
179    Some(MarkdownFence {
180        format: "yaml".to_string(),
181        block_start: 0,
182        inner_start,
183        inner_end,
184        block_end,
185    })
186}
187
188/// Index of the first line after the opener that holds a front-matter closer
189/// (`---` or `...`). Returns `None` if no closer is found.
190fn front_matter_closer(content: &str, lines: &[LineSpan]) -> Option<usize> {
191    // ponytail: skip the opener line (idx 0) and walk the rest with a numeric
192    // counter so we avoid the `.enumerate().skip(1)` chain shape used in jscpd-rs.
193    let mut idx = 1usize;
194    while idx < lines.len() {
195        let span = &lines[idx];
196        let line = content[span.start..span.end].trim();
197        if line == "---" || line == "..." {
198            return Some(idx);
199        }
200        idx += 1;
201    }
202    None
203}
204
205fn collect_ignore_byte_ranges(content: &str) -> Vec<[usize; 2]> {
206    let lines = line_spans(content);
207    let mut ranges = Vec::new();
208    let mut in_ignore = false;
209    let mut ignore_start: usize = 0;
210    for span in &lines {
211        let line = &content[span.start..span.end];
212        if line.contains("jscpd:ignore-start") {
213            in_ignore = true;
214            ignore_start = span.start;
215        } else if line.contains("jscpd:ignore-end") && in_ignore {
216            let end = span.next_start.min(content.len());
217            ranges.push([ignore_start, end]);
218            in_ignore = false;
219        }
220    }
221    ranges
222}
223
224pub fn tokens_to_detection(tokens: Vec<Token>, options: &TokenizeOptions) -> Vec<DetectionToken> {
225    let mut detection = Vec::with_capacity(tokens.len());
226    for t in tokens {
227        let byte_start = t.start.offset as usize;
228        let byte_end = t.end.offset as usize;
229        push_token(
230            &mut detection,
231            t.kind,
232            &t.value,
233            byte_start,
234            byte_end,
235            t.start,
236            t.end,
237            options,
238        );
239    }
240    detection
241}
242
243pub fn offset_detection_tokens(
244    tokens: &mut [DetectionToken],
245    byte_offset: usize,
246    start_location: &Location,
247) {
248    let line_offset = start_location.line.saturating_sub(1);
249    let col_offset = start_location.column;
250    for t in tokens.iter_mut() {
251        t.start.line += line_offset;
252        t.end.line += line_offset;
253        t.start.offset += byte_offset as u32;
254        t.end.offset += byte_offset as u32;
255        t.range[0] += byte_offset;
256        t.range[1] += byte_offset;
257        if t.start.line == start_location.line {
258            t.start.column += col_offset;
259        }
260        if t.end.line == start_location.line {
261            t.end.column += col_offset;
262        }
263    }
264}
265
266pub fn tokenize_markdown_maps(source: &str, options: &TokenizeOptions) -> Vec<TokenMap> {
267    if source.is_empty() {
268        return Vec::new();
269    }
270
271    let ignore_ranges = collect_ignore_byte_ranges(source);
272
273    let mut fences = extract_code_fences(source);
274    if let Some(fm) = extract_front_matter(source) {
275        fences.push(fm);
276        fences.sort_by_key(|f| f.block_start);
277    }
278
279    let sanitized = blank_ranges_preserve_newlines(
280        source,
281        &fences
282            .iter()
283            .map(|f| [f.block_start, f.block_end])
284            .collect::<Vec<_>>(),
285    );
286
287    let line_index = LineIndex::new(source.as_bytes());
288
289    let body_tokens = crate::generic::tokenize_generic(&sanitized, "markdown");
290    let mut markdown_detection = tokens_to_detection(body_tokens, options);
291    markdown_detection.retain(|t| t.range[0] < t.range[1]);
292
293    let mut maps = Vec::new();
294    if !markdown_detection.is_empty() {
295        maps.push(TokenMap {
296            format: "markdown".to_string(),
297            tokens: markdown_detection,
298        });
299    }
300
301    let mut embedded_maps: BTreeMap<String, Vec<DetectionToken>> = BTreeMap::new();
302    for fence in &fences {
303        let inner = &source[fence.inner_start..fence.inner_end];
304        let resolved = resolve_format(&fence.format).unwrap_or("text");
305        let outer_ignored = ignore_ranges
306            .iter()
307            .any(|[rs, re]| fence.inner_start < *re && fence.inner_end > *rs);
308
309        let mut inner_tokens = tokenize_format_to_detection(resolved, inner, options);
310
311        if outer_ignored {
312            for t in &mut inner_tokens {
313                t.range = [0, 0]; // mark as ignored by zeroing range
314            }
315            continue;
316        }
317
318        let inner_start_loc = line_index.location(fence.inner_start);
319        offset_detection_tokens(&mut inner_tokens, fence.inner_start, &inner_start_loc);
320
321        embedded_maps
322            .entry(resolved.to_string())
323            .or_default()
324            .extend(inner_tokens);
325    }
326
327    for (format, tokens) in embedded_maps {
328        maps.push(TokenMap { format, tokens });
329    }
330
331    maps
332}
333
334pub fn tokenize_markdown(source: &str, mode: Mode) -> Vec<Token> {
335    if source.is_empty() {
336        return Vec::new();
337    }
338
339    let mut in_ignore = false;
340    let mut ignore_ranges: Vec<(u32, u32)> = Vec::new();
341    let mut ignore_start = 0u32;
342
343    for (line_idx, line) in source.lines().enumerate() {
344        let line_num = line_idx as u32 + 1;
345        if line.contains("jscpd:ignore-start") {
346            in_ignore = true;
347            ignore_start = line_num;
348        } else if line.contains("jscpd:ignore-end") && in_ignore {
349            ignore_ranges.push((ignore_start, line_num));
350            in_ignore = false;
351        }
352    }
353
354    let fences = extract_fences(source);
355    let mut all_tokens = Vec::new();
356
357    // Prose body: blank out fenced code blocks and front matter, then
358    // tokenize the remainder as markdown — mirrors tokenize_markdown_maps so
359    // prose-only files (no code fences) still produce display tokens and are
360    // counted as analyzed.
361    let mut blanked_ranges: Vec<[usize; 2]> = extract_code_fences(source)
362        .iter()
363        .map(|f| [f.block_start, f.block_end])
364        .collect();
365    if let Some(fm) = extract_front_matter(source) {
366        blanked_ranges.push([fm.block_start, fm.block_end]);
367        blanked_ranges.sort_by_key(|r| r[0]);
368    }
369    let sanitized = blank_ranges_preserve_newlines(source, &blanked_ranges);
370    let mut body_tokens = crate::generic::tokenize_generic(&sanitized, "markdown");
371    for token in &mut body_tokens {
372        let in_outer_ignore = ignore_ranges
373            .iter()
374            .any(|(start, end)| token.start.line >= *start && token.start.line <= *end);
375        if in_outer_ignore {
376            token.kind = TokenKind::Ignore;
377        }
378    }
379    all_tokens.extend(body_tokens);
380
381    for fence in &fences {
382        let in_outer_ignore = ignore_ranges
383            .iter()
384            .any(|(start, end)| fence.start_line >= *start && fence.start_line <= *end);
385
386        let format = fence.language.as_deref().unwrap_or("text");
387        let mut fence_tokens = crate::tokenizer::tokenize(format, &fence.content, mode);
388
389        let line_offset = fence.start_line.saturating_sub(1);
390        for token in &mut fence_tokens {
391            token.start.line += line_offset;
392            token.end.line += line_offset;
393            if in_outer_ignore {
394                token.kind = TokenKind::Ignore;
395            }
396        }
397
398        all_tokens.extend(fence_tokens);
399    }
400
401    // Body and fence tokens are collected separately — restore source order.
402    all_tokens.sort_by_key(|t| (t.start.line, t.start.column));
403    all_tokens
404}
405
406// Legacy line-based fence extraction used by the display-path tokenize_markdown()
407struct CodeFence {
408    language: Option<String>,
409    content: String,
410    start_line: u32,
411}
412
413fn extract_fences(source: &str) -> Vec<CodeFence> {
414    let mut fences = Vec::new();
415    let mut in_fence = false;
416    let mut fence_char = '`';
417    let mut fence_lang: Option<String> = None;
418    let mut fence_content = String::new();
419    let mut fence_start_line = 0u32;
420
421    for (line_idx, line) in source.lines().enumerate() {
422        let line_num = line_idx as u32 + 1;
423        let trimmed = line.trim_start();
424
425        if !in_fence {
426            if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
427                let fc = trimmed.chars().next().unwrap_or('`');
428                let rest = trimmed.trim_start_matches(fc).trim();
429                fence_lang = if rest.is_empty() {
430                    None
431                } else {
432                    Some(rest.to_string())
433                };
434                fence_char = fc;
435                in_fence = true;
436                fence_content.clear();
437                fence_start_line = line_num + 1;
438            }
439        } else {
440            let close_trimmed = trimmed.trim_end();
441            if is_closing_fence_legacy(close_trimmed, fence_char) {
442                fences.push(CodeFence {
443                    language: fence_lang.take(),
444                    content: fence_content.clone(),
445                    start_line: fence_start_line,
446                });
447                fence_content.clear();
448                in_fence = false;
449            } else {
450                fence_content.push_str(line);
451                fence_content.push('\n');
452            }
453        }
454    }
455
456    fences
457}
458
459fn is_closing_fence_legacy(line: &str, fence_char: char) -> bool {
460    if !line.starts_with(fence_char) {
461        return false;
462    }
463    let count = line.chars().take_while(|&c| c == fence_char).count();
464    if count < 3 {
465        return false;
466    }
467    line.chars().skip(count).all(|c| c == ' ' || c == '\t')
468}
469
470#[cfg(test)]
471mod tests {
472    use super::*;
473    use crate::tokenizer::Mode;
474
475    // --- legacy display-path tests ---
476
477    const MD_WITH_JS: &str = "# Header\n\nSome prose.\n\n```javascript\nfunction hello() { return 42; }\n```\n\nMore prose.\n";
478    const MD_NO_FENCES: &str = "# Just a Header\n\nSome plain text with no code.\n";
479    const MD_UNKNOWN_LANG: &str = "```unknownlang999\nhello world\n```\n";
480    const MD_WITH_IGNORE: &str = "<!-- jscpd:ignore-start -->\n```javascript\nconst x = 1;\n```\n<!-- jscpd:ignore-end -->\n```javascript\nconst y = 2;\n```\n";
481
482    #[test]
483    fn js_fence_produces_tokens() {
484        let tokens = tokenize_markdown(MD_WITH_JS, Mode::Mild);
485        assert!(!tokens.is_empty(), "JS code fence must produce tokens");
486    }
487
488    #[test]
489    fn no_fences_produces_prose_tokens() {
490        let tokens = tokenize_markdown(MD_NO_FENCES, Mode::Mild);
491        assert!(
492            !tokens.is_empty(),
493            "Markdown with no fences must still produce prose tokens (issue #883)"
494        );
495    }
496
497    #[test]
498    fn unknown_lang_fence_does_not_panic() {
499        let result = std::panic::catch_unwind(|| tokenize_markdown(MD_UNKNOWN_LANG, Mode::Mild));
500        assert!(result.is_ok(), "unknown language fence must not panic");
501    }
502
503    #[test]
504    fn empty_markdown_returns_empty() {
505        let tokens = tokenize_markdown("", Mode::Mild);
506        assert!(tokens.is_empty());
507    }
508
509    #[test]
510    fn ignore_region_suppresses_fence_tokens() {
511        let tokens = tokenize_markdown(MD_WITH_IGNORE, Mode::Mild);
512        let non_ignore = tokens
513            .iter()
514            .filter(|t| t.kind != TokenKind::Ignore)
515            .count();
516        let ignore_count = tokens
517            .iter()
518            .filter(|t| t.kind == TokenKind::Ignore)
519            .count();
520        assert!(
521            ignore_count > 0,
522            "tokens in ignore region must be Ignore kind"
523        );
524        assert!(
525            non_ignore > 0,
526            "tokens outside ignore region must NOT be Ignore kind"
527        );
528    }
529
530    // --- tokenize_markdown_maps tests ---
531
532    fn default_options() -> TokenizeOptions {
533        TokenizeOptions::new(Mode::Mild)
534    }
535
536    fn find_map<'a>(maps: &'a [TokenMap], format: &str) -> Option<&'a TokenMap> {
537        maps.iter().find(|m| m.format == format)
538    }
539
540    fn assert_has_format(maps: &[TokenMap], format: &str, msg: &str) {
541        assert!(find_map(maps, format).is_some(), "{}", msg);
542    }
543
544    #[test]
545    fn maps_empty_source_returns_empty() {
546        let maps = tokenize_markdown_maps("", &default_options());
547        assert!(maps.is_empty());
548    }
549
550    #[test]
551    fn maps_js_fence_produces_javascript_entry() {
552        let source = "# Title\n\n```javascript\nfunction hello() { return 42; }\n```\n";
553        let maps = tokenize_markdown_maps(source, &default_options());
554        let js_map = find_map(&maps, "javascript");
555        assert!(js_map.is_some(), "must have a javascript TokenMap");
556        assert!(
557            !js_map.unwrap().tokens.is_empty(),
558            "javascript tokens must be non-empty"
559        );
560    }
561
562    #[test]
563    fn maps_multi_format_returns_separate_token_maps() {
564        let source = "```javascript\nvar x = 1;\n```\n```python\ny = 2\n```\n";
565        let maps = tokenize_markdown_maps(source, &default_options());
566        assert_has_format(&maps, "javascript", "must have javascript TokenMap");
567        assert_has_format(&maps, "python", "must have python TokenMap");
568    }
569
570    #[test]
571    fn maps_no_fences_produces_markdown_prose_only() {
572        let source = "# Just prose\n\nNo code here.\n";
573        let maps = tokenize_markdown_maps(source, &default_options());
574        assert_has_format(&maps, "markdown", "must have markdown TokenMap for prose");
575        assert!(
576            maps.iter().all(|m| m.format == "markdown"),
577            "no other formats expected"
578        );
579    }
580
581    #[test]
582    fn maps_unknown_language_skipped() {
583        let source = "```xyzunknown999\nhello world\n```\n";
584        let maps = tokenize_markdown_maps(source, &default_options());
585        assert!(
586            maps.iter().all(|m| m.format != "xyzunknown999"),
587            "unknown language should not produce its own format map"
588        );
589    }
590
591    #[test]
592    fn maps_tilde_fences_supported() {
593        let source = "~~~javascript\nconst x = 1;\n~~~\n";
594        let maps = tokenize_markdown_maps(source, &default_options());
595        assert!(
596            find_map(&maps, "javascript").is_some(),
597            "tilde fences must produce javascript TokenMap"
598        );
599    }
600
601    #[test]
602    fn maps_yaml_front_matter() {
603        let source = "---\ntitle: Hello\nauthor: World\n---\n\nSome prose.\n";
604        let maps = tokenize_markdown_maps(source, &default_options());
605        assert!(
606            find_map(&maps, "yaml").is_some(),
607            "YAML front matter must produce yaml TokenMap"
608        );
609    }
610
611    #[test]
612    fn maps_front_matter_with_ellipsis_terminator() {
613        let source = "---\ntitle: Hello\n...\n\nMore text.\n";
614        let maps = tokenize_markdown_maps(source, &default_options());
615        assert!(
616            find_map(&maps, "yaml").is_some(),
617            "... must terminate front matter as yaml"
618        );
619    }
620
621    #[test]
622    fn maps_front_matter_without_closing_is_prose() {
623        let source = "---\nthis is not front matter\nit has no closing marker\n";
624        let maps = tokenize_markdown_maps(source, &default_options());
625        assert!(
626            find_map(&maps, "yaml").is_none(),
627            "unclosed --- must not be treated as yaml"
628        );
629    }
630
631    #[test]
632    fn maps_ignore_region_suppresses_fence_tokens() {
633        let source = "<!-- jscpd:ignore-start -->\n```javascript\nconst x = 1;\n```\n<!-- jscpd:ignore-end -->\n```javascript\nconst y = 2;\n```\n";
634        let maps = tokenize_markdown_maps(source, &default_options());
635        let js_map = find_map(&maps, "javascript");
636        assert!(js_map.is_some(), "javascript map must exist");
637        let non_ignored_count = js_map.unwrap().tokens.len();
638        assert!(
639            non_ignored_count > 0,
640            "second fence must yield non-ignored tokens"
641        );
642    }
643
644    #[test]
645    fn maps_backtick_tilde_do_not_close_each_other() {
646        let source = "```javascript\nconst a = 1;\n~~~\nconst b = 2;\n```\n";
647        let maps = tokenize_markdown_maps(source, &default_options());
648        assert!(
649            find_map(&maps, "javascript").is_some(),
650            "backtick fence should not be closed by tilde"
651        );
652    }
653
654    #[test]
655    fn maps_closing_fence_length_must_match() {
656        let source = "````javascript\nconst x = 1;\n````\n";
657        let maps = tokenize_markdown_maps(source, &default_options());
658        assert_has_format(&maps, "javascript", "4-backtick fence must work");
659    }
660
661    #[test]
662    fn maps_fence_with_info_string_space() {
663        let source = "```javascript extra info\nconst x = 1;\n```\n";
664        let maps = tokenize_markdown_maps(source, &default_options());
665        assert!(
666            find_map(&maps, "javascript").is_some(),
667            "first whitespace-delimited token is the language"
668        );
669    }
670
671    #[test]
672    fn maps_returns_markdown_prose_tokens() {
673        let source = "# Header\n\nSome prose.\n\n```javascript\nvar x;\n```\n";
674        let maps = tokenize_markdown_maps(source, &default_options());
675        let md_map = find_map(&maps, "markdown");
676        assert!(md_map.is_some(), "must have markdown TokenMap for prose");
677        assert!(
678            !md_map.unwrap().tokens.is_empty(),
679            "prose must produce tokens"
680        );
681    }
682
683    #[test]
684    fn maps_detection_tokens_have_valid_positions() {
685        let source = "```javascript\nconst x = 1;\n```\n";
686        let maps = tokenize_markdown_maps(source, &default_options());
687        let js_map = find_map(&maps, "javascript");
688        assert!(js_map.is_some());
689        for t in &js_map.unwrap().tokens {
690            assert!(t.start.line >= 1, "line must be 1-based");
691            assert!(t.start.offset as i32 >= 0, "offset must be non-negative");
692        }
693    }
694
695    #[test]
696    fn line_spans_basic() {
697        let content = "hello\nworld\n";
698        let spans = line_spans(content);
699        assert_eq!(spans.len(), 3);
700        assert_eq!(&content[spans[0].start..spans[0].end], "hello");
701        assert_eq!(&content[spans[1].start..spans[1].end], "world");
702    }
703
704    #[test]
705    fn line_spans_empty_content() {
706        let spans = line_spans("");
707        assert!(spans.is_empty());
708    }
709
710    #[test]
711    fn line_spans_no_trailing_newline() {
712        let content = "one\ntwo";
713        let spans = line_spans(content);
714        assert_eq!(spans.len(), 2);
715        assert_eq!(&content[spans[0].start..spans[0].end], "one");
716        assert_eq!(&content[spans[1].start..spans[1].end], "two");
717    }
718
719    #[test]
720    fn opening_fence_detection() {
721        assert!(parse_opening_fence("```javascript").is_some());
722        assert!(parse_opening_fence("~~~python").is_some());
723        assert!(parse_opening_fence("``").is_none());
724        assert!(parse_opening_fence("not a fence").is_none());
725    }
726
727    #[test]
728    fn closing_fence_detection() {
729        let open = FenceOpen {
730            marker: b'`',
731            len: 3,
732            info: String::new(),
733        };
734        assert!(is_closing_fence("```", &open));
735        assert!(is_closing_fence("````", &open));
736        assert!(!is_closing_fence("~~", &open));
737        assert!(!is_closing_fence("```javascript", &open));
738    }
739
740    #[test]
741    fn byte_offsets_are_correct_for_front_matter() {
742        let source = "---\ntitle: Hello\n---\n\nText.\n";
743        let fm = extract_front_matter(source).unwrap();
744        assert_eq!(fm.format, "yaml");
745        assert_eq!(fm.block_start, 0);
746        assert_eq!(&source[fm.inner_start..fm.inner_end], "title: Hello");
747    }
748
749    #[test]
750    fn byte_offsets_are_correct_for_code_block() {
751        let source = "# Header\n\n```javascript\nconst x = 1;\n```\n";
752        let fences = extract_code_fences(source);
753        assert_eq!(fences.len(), 1);
754        let f = &fences[0];
755        assert_eq!(f.format, "javascript");
756        let inner = &source[f.inner_start..f.inner_end];
757        assert!(inner.contains("const x = 1;"));
758    }
759
760    #[test]
761    fn resolve_format_js() {
762        assert_eq!(resolve_fence_format("javascript"), Some("javascript"));
763    }
764
765    #[test]
766    fn resolve_format_unknown() {
767        assert!(resolve_fence_format("xyzunknown999").is_none());
768    }
769
770    #[test]
771    fn maps_synonym_resolution() {
772        let source = "```node\nconst x = 1;\n```\n";
773        let maps = tokenize_markdown_maps(source, &default_options());
774        assert!(
775            find_map(&maps, "javascript").is_some(),
776            "node must resolve to javascript"
777        );
778    }
779
780    #[test]
781    fn maps_shell_resolves_to_bash() {
782        let source = "```shell\necho hello\n```\n";
783        let maps = tokenize_markdown_maps(source, &default_options());
784        assert!(
785            find_map(&maps, "bash").is_some(),
786            "shell must resolve to bash"
787        );
788    }
789}