1use std::collections::BTreeMap;
4
5use cpd_core::models::{DetectionToken, Location, Token, TokenKind};
6
7use crate::embedded::blank_ranges_preserve_newlines;
8use crate::formats::resolve_format;
9use crate::line_index::LineIndex;
10use crate::tokenizer::{Mode, TokenMap, TokenizeOptions, push_token, tokenize_format_to_detection};
11
12pub struct LineSpan {
13 pub start: usize,
14 pub end: usize,
15 pub next_start: usize,
16}
17
18pub fn line_spans(content: &str) -> Vec<LineSpan> {
19 if content.is_empty() {
20 return Vec::new();
21 }
22 let mut spans = Vec::new();
23 let bytes = content.as_bytes();
24 let len = bytes.len();
25 let mut pos = 0usize;
26 while pos <= len {
27 let line_end = bytes[pos..]
28 .iter()
29 .position(|&b| b == b'\n')
30 .map(|i| pos + i)
31 .unwrap_or(len);
32 let content_end = if line_end > pos && bytes[line_end - 1] == b'\r' {
33 line_end - 1
34 } else {
35 line_end
36 };
37 let next_start = if line_end < len { line_end + 1 } else { len };
38 spans.push(LineSpan {
39 start: pos,
40 end: content_end,
41 next_start,
42 });
43 if next_start <= pos {
44 break;
45 }
46 pos = next_start;
47 if pos >= len && (len == 0 || bytes[len - 1] != b'\n') {
48 break;
49 }
50 }
51 spans
52}
53
54#[derive(Debug, Clone)]
55struct MarkdownFence {
56 format: String,
57 block_start: usize,
58 inner_start: usize,
59 inner_end: usize,
60 block_end: usize,
61}
62
63struct FenceOpen {
64 marker: u8,
65 len: usize,
66 info: String,
67}
68
69fn parse_opening_fence(line: &str) -> Option<FenceOpen> {
70 let (&first, rest) = line.as_bytes().split_first()?;
72 let marker = first;
73 if !matches!(marker, b'`' | b'~') {
74 return None;
75 }
76 let len = 1 + rest.iter().take_while(|&&b| b == marker).count();
77 if len < 3 {
78 return None;
79 }
80 Some(FenceOpen {
81 marker,
82 len,
83 info: line[len..].trim().to_string(),
84 })
85}
86
87fn is_closing_fence(line: &str, open: &FenceOpen) -> bool {
88 let trimmed = line.trim();
89 let bytes = trimmed.as_bytes();
90 if bytes.is_empty() {
91 return false;
92 }
93 let len = bytes.iter().take_while(|&&b| b == open.marker).count();
94 len >= open.len && bytes[len..].iter().all(|&b| b == b' ' || b == b'\t')
95}
96
97fn resolve_fence_format(info: &str) -> Option<&'static str> {
98 let tag = info.split_whitespace().next()?;
99 resolve_format(tag)
100}
101
102fn fence_bounds(
103 content: &str,
104 lines: &[LineSpan],
105 close_idx: usize,
106 inner_start: usize,
107) -> (usize, usize) {
108 let inner_end = content[..lines[close_idx].start]
109 .strip_suffix('\n')
110 .map(|prefix| prefix.len())
111 .unwrap_or(lines[close_idx].start)
112 .max(inner_start);
113 let block_end = lines[close_idx].next_start.min(content.len());
114 (inner_end, block_end)
115}
116
117fn extract_code_fences(content: &str) -> Vec<MarkdownFence> {
118 let lines = line_spans(content);
119 let mut fences = Vec::new();
120 let mut idx = 0usize;
121 while idx < lines.len() {
122 let line_text = &content[lines[idx].start..lines[idx].end];
123 let Some(open) = parse_opening_fence(line_text) else {
124 idx += 1;
125 continue;
126 };
127 let resolved = resolve_fence_format(&open.info);
128 let close_idx = lines[idx + 1..]
129 .iter()
130 .position(|span| {
131 let candidate = &content[span.start..span.end];
132 is_closing_fence(candidate, &open)
133 })
134 .map(|p| idx + 1 + p);
135 let Some(close_idx) = close_idx else {
136 idx += 1;
137 continue;
138 };
139 let inner_start = lines
140 .get(idx + 1)
141 .map(|s| s.start)
142 .unwrap_or(lines[idx].next_start);
143 let (inner_end, block_end) = fence_bounds(content, &lines, close_idx, inner_start);
144 let format = resolved.map(|r| r.to_string()).unwrap_or_else(|| {
145 open.info
146 .split_whitespace()
147 .next()
148 .unwrap_or("")
149 .to_string()
150 });
151 fences.push(MarkdownFence {
152 format,
153 block_start: lines[idx].start,
154 inner_start,
155 inner_end,
156 block_end,
157 });
158 idx = close_idx + 1;
159 }
160 fences
161}
162
163fn extract_front_matter(content: &str) -> Option<MarkdownFence> {
164 let opener_len = if content.starts_with("---\r\n") {
167 5
168 } else if content.starts_with("---\n") {
169 4
170 } else {
171 return None;
172 };
173
174 let lines = line_spans(content);
175 let inner_start = opener_len;
177 let close_idx = front_matter_closer(content, &lines)?;
178 let (inner_end, block_end) = fence_bounds(content, &lines, close_idx, inner_start);
179 Some(MarkdownFence {
180 format: "yaml".to_string(),
181 block_start: 0,
182 inner_start,
183 inner_end,
184 block_end,
185 })
186}
187
188fn front_matter_closer(content: &str, lines: &[LineSpan]) -> Option<usize> {
191 let mut idx = 1usize;
194 while idx < lines.len() {
195 let span = &lines[idx];
196 let line = content[span.start..span.end].trim();
197 if line == "---" || line == "..." {
198 return Some(idx);
199 }
200 idx += 1;
201 }
202 None
203}
204
205fn collect_ignore_byte_ranges(content: &str) -> Vec<[usize; 2]> {
206 let lines = line_spans(content);
207 let mut ranges = Vec::new();
208 let mut in_ignore = false;
209 let mut ignore_start: usize = 0;
210 for span in &lines {
211 let line = &content[span.start..span.end];
212 if line.contains("jscpd:ignore-start") {
213 in_ignore = true;
214 ignore_start = span.start;
215 } else if line.contains("jscpd:ignore-end") && in_ignore {
216 let end = span.next_start.min(content.len());
217 ranges.push([ignore_start, end]);
218 in_ignore = false;
219 }
220 }
221 ranges
222}
223
224pub fn tokens_to_detection(tokens: Vec<Token>, options: &TokenizeOptions) -> Vec<DetectionToken> {
225 let mut detection = Vec::with_capacity(tokens.len());
226 for t in tokens {
227 let byte_start = t.start.offset as usize;
228 let byte_end = t.end.offset as usize;
229 push_token(
230 &mut detection,
231 t.kind,
232 &t.value,
233 byte_start,
234 byte_end,
235 t.start,
236 t.end,
237 options,
238 );
239 }
240 detection
241}
242
243pub fn offset_detection_tokens(
244 tokens: &mut [DetectionToken],
245 byte_offset: usize,
246 start_location: &Location,
247) {
248 let line_offset = start_location.line.saturating_sub(1);
249 let col_offset = start_location.column;
250 for t in tokens.iter_mut() {
251 t.start.line += line_offset;
252 t.end.line += line_offset;
253 t.start.offset += byte_offset as u32;
254 t.end.offset += byte_offset as u32;
255 t.range[0] += byte_offset;
256 t.range[1] += byte_offset;
257 if t.start.line == start_location.line {
258 t.start.column += col_offset;
259 }
260 if t.end.line == start_location.line {
261 t.end.column += col_offset;
262 }
263 }
264}
265
266pub fn tokenize_markdown_maps(source: &str, options: &TokenizeOptions) -> Vec<TokenMap> {
267 if source.is_empty() {
268 return Vec::new();
269 }
270
271 let ignore_ranges = collect_ignore_byte_ranges(source);
272
273 let mut fences = extract_code_fences(source);
274 if let Some(fm) = extract_front_matter(source) {
275 fences.push(fm);
276 fences.sort_by_key(|f| f.block_start);
277 }
278
279 let sanitized = blank_ranges_preserve_newlines(
280 source,
281 &fences
282 .iter()
283 .map(|f| [f.block_start, f.block_end])
284 .collect::<Vec<_>>(),
285 );
286
287 let line_index = LineIndex::new(source.as_bytes());
288
289 let body_tokens = crate::generic::tokenize_generic(&sanitized, "markdown");
290 let mut markdown_detection = tokens_to_detection(body_tokens, options);
291 markdown_detection.retain(|t| t.range[0] < t.range[1]);
292
293 let mut maps = Vec::new();
294 if !markdown_detection.is_empty() {
295 maps.push(TokenMap {
296 format: "markdown".to_string(),
297 tokens: markdown_detection,
298 });
299 }
300
301 let mut embedded_maps: BTreeMap<String, Vec<DetectionToken>> = BTreeMap::new();
302 for fence in &fences {
303 let inner = &source[fence.inner_start..fence.inner_end];
304 let resolved = resolve_format(&fence.format).unwrap_or("text");
305 let outer_ignored = ignore_ranges
306 .iter()
307 .any(|[rs, re]| fence.inner_start < *re && fence.inner_end > *rs);
308
309 let mut inner_tokens = tokenize_format_to_detection(resolved, inner, options);
310
311 if outer_ignored {
312 for t in &mut inner_tokens {
313 t.range = [0, 0]; }
315 continue;
316 }
317
318 let inner_start_loc = line_index.location(fence.inner_start);
319 offset_detection_tokens(&mut inner_tokens, fence.inner_start, &inner_start_loc);
320
321 embedded_maps
322 .entry(resolved.to_string())
323 .or_default()
324 .extend(inner_tokens);
325 }
326
327 for (format, tokens) in embedded_maps {
328 maps.push(TokenMap { format, tokens });
329 }
330
331 maps
332}
333
334pub fn tokenize_markdown(source: &str, mode: Mode) -> Vec<Token> {
335 if source.is_empty() {
336 return Vec::new();
337 }
338
339 let mut in_ignore = false;
340 let mut ignore_ranges: Vec<(u32, u32)> = Vec::new();
341 let mut ignore_start = 0u32;
342
343 for (line_idx, line) in source.lines().enumerate() {
344 let line_num = line_idx as u32 + 1;
345 if line.contains("jscpd:ignore-start") {
346 in_ignore = true;
347 ignore_start = line_num;
348 } else if line.contains("jscpd:ignore-end") && in_ignore {
349 ignore_ranges.push((ignore_start, line_num));
350 in_ignore = false;
351 }
352 }
353
354 let fences = extract_fences(source);
355 let mut all_tokens = Vec::new();
356
357 let mut blanked_ranges: Vec<[usize; 2]> = extract_code_fences(source)
362 .iter()
363 .map(|f| [f.block_start, f.block_end])
364 .collect();
365 if let Some(fm) = extract_front_matter(source) {
366 blanked_ranges.push([fm.block_start, fm.block_end]);
367 blanked_ranges.sort_by_key(|r| r[0]);
368 }
369 let sanitized = blank_ranges_preserve_newlines(source, &blanked_ranges);
370 let mut body_tokens = crate::generic::tokenize_generic(&sanitized, "markdown");
371 for token in &mut body_tokens {
372 let in_outer_ignore = ignore_ranges
373 .iter()
374 .any(|(start, end)| token.start.line >= *start && token.start.line <= *end);
375 if in_outer_ignore {
376 token.kind = TokenKind::Ignore;
377 }
378 }
379 all_tokens.extend(body_tokens);
380
381 for fence in &fences {
382 let in_outer_ignore = ignore_ranges
383 .iter()
384 .any(|(start, end)| fence.start_line >= *start && fence.start_line <= *end);
385
386 let format = fence.language.as_deref().unwrap_or("text");
387 let mut fence_tokens = crate::tokenizer::tokenize(format, &fence.content, mode);
388
389 let line_offset = fence.start_line.saturating_sub(1);
390 for token in &mut fence_tokens {
391 token.start.line += line_offset;
392 token.end.line += line_offset;
393 if in_outer_ignore {
394 token.kind = TokenKind::Ignore;
395 }
396 }
397
398 all_tokens.extend(fence_tokens);
399 }
400
401 all_tokens.sort_by_key(|t| (t.start.line, t.start.column));
403 all_tokens
404}
405
406struct CodeFence {
408 language: Option<String>,
409 content: String,
410 start_line: u32,
411}
412
413fn extract_fences(source: &str) -> Vec<CodeFence> {
414 let mut fences = Vec::new();
415 let mut in_fence = false;
416 let mut fence_char = '`';
417 let mut fence_lang: Option<String> = None;
418 let mut fence_content = String::new();
419 let mut fence_start_line = 0u32;
420
421 for (line_idx, line) in source.lines().enumerate() {
422 let line_num = line_idx as u32 + 1;
423 let trimmed = line.trim_start();
424
425 if !in_fence {
426 if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
427 let fc = trimmed.chars().next().unwrap_or('`');
428 let rest = trimmed.trim_start_matches(fc).trim();
429 fence_lang = if rest.is_empty() {
430 None
431 } else {
432 Some(rest.to_string())
433 };
434 fence_char = fc;
435 in_fence = true;
436 fence_content.clear();
437 fence_start_line = line_num + 1;
438 }
439 } else {
440 let close_trimmed = trimmed.trim_end();
441 if is_closing_fence_legacy(close_trimmed, fence_char) {
442 fences.push(CodeFence {
443 language: fence_lang.take(),
444 content: fence_content.clone(),
445 start_line: fence_start_line,
446 });
447 fence_content.clear();
448 in_fence = false;
449 } else {
450 fence_content.push_str(line);
451 fence_content.push('\n');
452 }
453 }
454 }
455
456 fences
457}
458
459fn is_closing_fence_legacy(line: &str, fence_char: char) -> bool {
460 if !line.starts_with(fence_char) {
461 return false;
462 }
463 let count = line.chars().take_while(|&c| c == fence_char).count();
464 if count < 3 {
465 return false;
466 }
467 line.chars().skip(count).all(|c| c == ' ' || c == '\t')
468}
469
470#[cfg(test)]
471mod tests {
472 use super::*;
473 use crate::tokenizer::Mode;
474
475 const MD_WITH_JS: &str = "# Header\n\nSome prose.\n\n```javascript\nfunction hello() { return 42; }\n```\n\nMore prose.\n";
478 const MD_NO_FENCES: &str = "# Just a Header\n\nSome plain text with no code.\n";
479 const MD_UNKNOWN_LANG: &str = "```unknownlang999\nhello world\n```\n";
480 const MD_WITH_IGNORE: &str = "<!-- jscpd:ignore-start -->\n```javascript\nconst x = 1;\n```\n<!-- jscpd:ignore-end -->\n```javascript\nconst y = 2;\n```\n";
481
482 #[test]
483 fn js_fence_produces_tokens() {
484 let tokens = tokenize_markdown(MD_WITH_JS, Mode::Mild);
485 assert!(!tokens.is_empty(), "JS code fence must produce tokens");
486 }
487
488 #[test]
489 fn no_fences_produces_prose_tokens() {
490 let tokens = tokenize_markdown(MD_NO_FENCES, Mode::Mild);
491 assert!(
492 !tokens.is_empty(),
493 "Markdown with no fences must still produce prose tokens (issue #883)"
494 );
495 }
496
497 #[test]
498 fn unknown_lang_fence_does_not_panic() {
499 let result = std::panic::catch_unwind(|| tokenize_markdown(MD_UNKNOWN_LANG, Mode::Mild));
500 assert!(result.is_ok(), "unknown language fence must not panic");
501 }
502
503 #[test]
504 fn empty_markdown_returns_empty() {
505 let tokens = tokenize_markdown("", Mode::Mild);
506 assert!(tokens.is_empty());
507 }
508
509 #[test]
510 fn ignore_region_suppresses_fence_tokens() {
511 let tokens = tokenize_markdown(MD_WITH_IGNORE, Mode::Mild);
512 let non_ignore = tokens
513 .iter()
514 .filter(|t| t.kind != TokenKind::Ignore)
515 .count();
516 let ignore_count = tokens
517 .iter()
518 .filter(|t| t.kind == TokenKind::Ignore)
519 .count();
520 assert!(
521 ignore_count > 0,
522 "tokens in ignore region must be Ignore kind"
523 );
524 assert!(
525 non_ignore > 0,
526 "tokens outside ignore region must NOT be Ignore kind"
527 );
528 }
529
530 fn default_options() -> TokenizeOptions {
533 TokenizeOptions::new(Mode::Mild)
534 }
535
536 fn find_map<'a>(maps: &'a [TokenMap], format: &str) -> Option<&'a TokenMap> {
537 maps.iter().find(|m| m.format == format)
538 }
539
540 fn assert_has_format(maps: &[TokenMap], format: &str, msg: &str) {
541 assert!(find_map(maps, format).is_some(), "{}", msg);
542 }
543
544 #[test]
545 fn maps_empty_source_returns_empty() {
546 let maps = tokenize_markdown_maps("", &default_options());
547 assert!(maps.is_empty());
548 }
549
550 #[test]
551 fn maps_js_fence_produces_javascript_entry() {
552 let source = "# Title\n\n```javascript\nfunction hello() { return 42; }\n```\n";
553 let maps = tokenize_markdown_maps(source, &default_options());
554 let js_map = find_map(&maps, "javascript");
555 assert!(js_map.is_some(), "must have a javascript TokenMap");
556 assert!(
557 !js_map.unwrap().tokens.is_empty(),
558 "javascript tokens must be non-empty"
559 );
560 }
561
562 #[test]
563 fn maps_multi_format_returns_separate_token_maps() {
564 let source = "```javascript\nvar x = 1;\n```\n```python\ny = 2\n```\n";
565 let maps = tokenize_markdown_maps(source, &default_options());
566 assert_has_format(&maps, "javascript", "must have javascript TokenMap");
567 assert_has_format(&maps, "python", "must have python TokenMap");
568 }
569
570 #[test]
571 fn maps_no_fences_produces_markdown_prose_only() {
572 let source = "# Just prose\n\nNo code here.\n";
573 let maps = tokenize_markdown_maps(source, &default_options());
574 assert_has_format(&maps, "markdown", "must have markdown TokenMap for prose");
575 assert!(
576 maps.iter().all(|m| m.format == "markdown"),
577 "no other formats expected"
578 );
579 }
580
581 #[test]
582 fn maps_unknown_language_skipped() {
583 let source = "```xyzunknown999\nhello world\n```\n";
584 let maps = tokenize_markdown_maps(source, &default_options());
585 assert!(
586 maps.iter().all(|m| m.format != "xyzunknown999"),
587 "unknown language should not produce its own format map"
588 );
589 }
590
591 #[test]
592 fn maps_tilde_fences_supported() {
593 let source = "~~~javascript\nconst x = 1;\n~~~\n";
594 let maps = tokenize_markdown_maps(source, &default_options());
595 assert!(
596 find_map(&maps, "javascript").is_some(),
597 "tilde fences must produce javascript TokenMap"
598 );
599 }
600
601 #[test]
602 fn maps_yaml_front_matter() {
603 let source = "---\ntitle: Hello\nauthor: World\n---\n\nSome prose.\n";
604 let maps = tokenize_markdown_maps(source, &default_options());
605 assert!(
606 find_map(&maps, "yaml").is_some(),
607 "YAML front matter must produce yaml TokenMap"
608 );
609 }
610
611 #[test]
612 fn maps_front_matter_with_ellipsis_terminator() {
613 let source = "---\ntitle: Hello\n...\n\nMore text.\n";
614 let maps = tokenize_markdown_maps(source, &default_options());
615 assert!(
616 find_map(&maps, "yaml").is_some(),
617 "... must terminate front matter as yaml"
618 );
619 }
620
621 #[test]
622 fn maps_front_matter_without_closing_is_prose() {
623 let source = "---\nthis is not front matter\nit has no closing marker\n";
624 let maps = tokenize_markdown_maps(source, &default_options());
625 assert!(
626 find_map(&maps, "yaml").is_none(),
627 "unclosed --- must not be treated as yaml"
628 );
629 }
630
631 #[test]
632 fn maps_ignore_region_suppresses_fence_tokens() {
633 let source = "<!-- jscpd:ignore-start -->\n```javascript\nconst x = 1;\n```\n<!-- jscpd:ignore-end -->\n```javascript\nconst y = 2;\n```\n";
634 let maps = tokenize_markdown_maps(source, &default_options());
635 let js_map = find_map(&maps, "javascript");
636 assert!(js_map.is_some(), "javascript map must exist");
637 let non_ignored_count = js_map.unwrap().tokens.len();
638 assert!(
639 non_ignored_count > 0,
640 "second fence must yield non-ignored tokens"
641 );
642 }
643
644 #[test]
645 fn maps_backtick_tilde_do_not_close_each_other() {
646 let source = "```javascript\nconst a = 1;\n~~~\nconst b = 2;\n```\n";
647 let maps = tokenize_markdown_maps(source, &default_options());
648 assert!(
649 find_map(&maps, "javascript").is_some(),
650 "backtick fence should not be closed by tilde"
651 );
652 }
653
654 #[test]
655 fn maps_closing_fence_length_must_match() {
656 let source = "````javascript\nconst x = 1;\n````\n";
657 let maps = tokenize_markdown_maps(source, &default_options());
658 assert_has_format(&maps, "javascript", "4-backtick fence must work");
659 }
660
661 #[test]
662 fn maps_fence_with_info_string_space() {
663 let source = "```javascript extra info\nconst x = 1;\n```\n";
664 let maps = tokenize_markdown_maps(source, &default_options());
665 assert!(
666 find_map(&maps, "javascript").is_some(),
667 "first whitespace-delimited token is the language"
668 );
669 }
670
671 #[test]
672 fn maps_returns_markdown_prose_tokens() {
673 let source = "# Header\n\nSome prose.\n\n```javascript\nvar x;\n```\n";
674 let maps = tokenize_markdown_maps(source, &default_options());
675 let md_map = find_map(&maps, "markdown");
676 assert!(md_map.is_some(), "must have markdown TokenMap for prose");
677 assert!(
678 !md_map.unwrap().tokens.is_empty(),
679 "prose must produce tokens"
680 );
681 }
682
683 #[test]
684 fn maps_detection_tokens_have_valid_positions() {
685 let source = "```javascript\nconst x = 1;\n```\n";
686 let maps = tokenize_markdown_maps(source, &default_options());
687 let js_map = find_map(&maps, "javascript");
688 assert!(js_map.is_some());
689 for t in &js_map.unwrap().tokens {
690 assert!(t.start.line >= 1, "line must be 1-based");
691 assert!(t.start.offset as i32 >= 0, "offset must be non-negative");
692 }
693 }
694
695 #[test]
696 fn line_spans_basic() {
697 let content = "hello\nworld\n";
698 let spans = line_spans(content);
699 assert_eq!(spans.len(), 3);
700 assert_eq!(&content[spans[0].start..spans[0].end], "hello");
701 assert_eq!(&content[spans[1].start..spans[1].end], "world");
702 }
703
704 #[test]
705 fn line_spans_empty_content() {
706 let spans = line_spans("");
707 assert!(spans.is_empty());
708 }
709
710 #[test]
711 fn line_spans_no_trailing_newline() {
712 let content = "one\ntwo";
713 let spans = line_spans(content);
714 assert_eq!(spans.len(), 2);
715 assert_eq!(&content[spans[0].start..spans[0].end], "one");
716 assert_eq!(&content[spans[1].start..spans[1].end], "two");
717 }
718
719 #[test]
720 fn opening_fence_detection() {
721 assert!(parse_opening_fence("```javascript").is_some());
722 assert!(parse_opening_fence("~~~python").is_some());
723 assert!(parse_opening_fence("``").is_none());
724 assert!(parse_opening_fence("not a fence").is_none());
725 }
726
727 #[test]
728 fn closing_fence_detection() {
729 let open = FenceOpen {
730 marker: b'`',
731 len: 3,
732 info: String::new(),
733 };
734 assert!(is_closing_fence("```", &open));
735 assert!(is_closing_fence("````", &open));
736 assert!(!is_closing_fence("~~", &open));
737 assert!(!is_closing_fence("```javascript", &open));
738 }
739
740 #[test]
741 fn byte_offsets_are_correct_for_front_matter() {
742 let source = "---\ntitle: Hello\n---\n\nText.\n";
743 let fm = extract_front_matter(source).unwrap();
744 assert_eq!(fm.format, "yaml");
745 assert_eq!(fm.block_start, 0);
746 assert_eq!(&source[fm.inner_start..fm.inner_end], "title: Hello");
747 }
748
749 #[test]
750 fn byte_offsets_are_correct_for_code_block() {
751 let source = "# Header\n\n```javascript\nconst x = 1;\n```\n";
752 let fences = extract_code_fences(source);
753 assert_eq!(fences.len(), 1);
754 let f = &fences[0];
755 assert_eq!(f.format, "javascript");
756 let inner = &source[f.inner_start..f.inner_end];
757 assert!(inner.contains("const x = 1;"));
758 }
759
760 #[test]
761 fn resolve_format_js() {
762 assert_eq!(resolve_fence_format("javascript"), Some("javascript"));
763 }
764
765 #[test]
766 fn resolve_format_unknown() {
767 assert!(resolve_fence_format("xyzunknown999").is_none());
768 }
769
770 #[test]
771 fn maps_synonym_resolution() {
772 let source = "```node\nconst x = 1;\n```\n";
773 let maps = tokenize_markdown_maps(source, &default_options());
774 assert!(
775 find_map(&maps, "javascript").is_some(),
776 "node must resolve to javascript"
777 );
778 }
779
780 #[test]
781 fn maps_shell_resolves_to_bash() {
782 let source = "```shell\necho hello\n```\n";
783 let maps = tokenize_markdown_maps(source, &default_options());
784 assert!(
785 find_map(&maps, "bash").is_some(),
786 "shell must resolve to bash"
787 );
788 }
789}