1use std::{
8 borrow::{Borrow, Cow},
9 collections::{BTreeMap, BTreeSet},
10};
11
12use num_bigint::BigUint;
13use pulldown_cmark::{Event, HeadingLevel, Options as CommonMarkOptions, Parser, Tag};
14use saphyr_parser::{
15 Event as ExactEvent, Marker, Parser as ExactParser, ScalarStyle, ScanError, Span, StrInput,
16 Tag as YamlTag,
17};
18
19use crate::{ByteOffset, HeaderLevel, TextRange};
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub struct MarkdownOptions {
24 pub strip_inline_markup: bool,
27}
28
29impl Default for MarkdownOptions {
30 fn default() -> Self {
31 Self {
32 strip_inline_markup: true,
33 }
34 }
35}
36
37#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct Document {
40 pub frontmatter: DocumentFrontmatter,
42 pub sections: Vec<Section>,
44 pub file_suppressions: Suppressions,
46}
47
48#[derive(Debug, Clone, PartialEq, Eq)]
50#[non_exhaustive]
51pub enum DocumentFrontmatter {
52 Absent,
54 Mapping {
56 value: serde_json::Map<String, serde_json::Value>,
58 location: FrontmatterLocation,
60 anchors: FrontmatterAnchors,
66 },
67 Invalid {
69 location: FrontmatterLocation,
72 message: String,
74 },
75}
76
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
79pub struct FrontmatterLocation {
80 pub range: TextRange,
82 pub start_line: u64,
84 pub end_line: u64,
86}
87
88#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
90pub struct FrontmatterAnchor {
91 pub line: u64,
94 pub column: u64,
96}
97
98#[derive(Debug, Clone, Default, PartialEq, Eq)]
110pub struct FrontmatterAnchors(BTreeMap<String, FrontmatterAnchor>);
111
112impl FrontmatterAnchors {
113 pub fn get(&self, pointer: &str) -> Option<FrontmatterAnchor> {
115 self.0.get(pointer).copied()
116 }
117
118 pub fn is_empty(&self) -> bool {
120 self.0.is_empty()
121 }
122}
123
124#[derive(Debug, Clone, PartialEq, Eq)]
126pub struct Section {
127 pub heading: Heading,
129 pub children: Vec<Section>,
135}
136
137#[derive(Debug, Clone, PartialEq, Eq)]
139pub struct Heading {
140 pub level: HeaderLevel,
142 pub text: String,
144 pub diagnostic_text: String,
148 pub source_text: String,
152 pub location: HeadingLocation,
154 pub suppressions: Suppressions,
156}
157
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
160pub struct HeadingLocation {
161 pub range: TextRange,
163 pub line_range: TextRange,
165 pub line: u64,
167 pub column: u64,
171}
172
173#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
175#[repr(transparent)]
176pub struct SuppressedDiagnostic(pub String);
177
178impl Borrow<str> for SuppressedDiagnostic {
179 fn borrow(&self) -> &str {
180 &self.0
181 }
182}
183
184#[derive(Debug, Clone, Default, PartialEq, Eq)]
186#[repr(transparent)]
187pub struct Suppressions(pub BTreeSet<SuppressedDiagnostic>);
188
189impl Suppressions {
190 pub fn contains(&self, id: &str) -> bool {
192 self.0.contains(id)
193 }
194}
195
196pub fn parse_markdown(source: &str, options: MarkdownOptions) -> Document {
215 let line_index = LineIndex::new(source);
216 let (frontmatter, frontmatter_range) = parse_frontmatter(source, &line_index);
217 let masked_source = frontmatter_range.map(|range| mask_source_range(source, range));
220 let parser_source = normalize_bare_cr(masked_source.as_deref().unwrap_or(source));
221 let mut headings = Vec::new();
222 let mut file_suppressions = Suppressions::default();
223 let mut line_suppressions = BTreeMap::new();
224 let mut active_heading: Option<HeadingBuilder> = None;
225 let mut container_depth = 0_usize;
226
227 for (event, range) in
228 Parser::new_ext(&parser_source, CommonMarkOptions::empty()).into_offset_iter()
229 {
230 match event {
231 Event::Start(Tag::BlockQuote(_) | Tag::List(_) | Tag::Item) => {
232 container_depth += 1;
233 }
234 Event::End(
235 pulldown_cmark::TagEnd::BlockQuote(_)
236 | pulldown_cmark::TagEnd::List(_)
237 | pulldown_cmark::TagEnd::Item,
238 ) => {
239 container_depth -= 1;
240 }
241 Event::Start(Tag::Heading { level, .. }) => {
242 active_heading = (container_depth == 0
243 && is_eligible_heading(source, &range, level, &line_index))
244 .then(|| HeadingBuilder::new(level, range));
245 }
246 Event::End(pulldown_cmark::TagEnd::Heading(_)) => {
247 if let Some(builder) = active_heading.take() {
248 headings.push(builder.finish(source, options, &line_index, &line_suppressions));
249 }
250 }
251 Event::Text(text) => {
252 if let Some(builder) = active_heading.as_mut() {
253 builder.push_visible(&text);
254 }
255 }
256 Event::Code(text) | Event::InlineMath(text) | Event::DisplayMath(text) => {
257 if let Some(builder) = active_heading.as_mut() {
258 builder.push_visible(&text);
259 }
260 }
261 Event::SoftBreak | Event::HardBreak => {
262 if let Some(builder) = active_heading.as_mut() {
263 builder.push_visible("\n");
264 }
265 }
266 Event::Html(html) | Event::InlineHtml(html) => {
267 collect_suppressions(
268 source,
269 &html,
270 range,
271 &line_index,
272 &mut file_suppressions,
273 &mut line_suppressions,
274 );
275 }
276 _ => {}
277 }
278 }
279
280 Document {
281 frontmatter,
282 sections: build_section_tree(headings),
283 file_suppressions,
284 }
285}
286
287fn parse_frontmatter(
288 source: &str,
289 lines: &LineIndex,
290) -> (DocumentFrontmatter, Option<std::ops::Range<usize>>) {
291 if lines.line_text(source, 1) != Some("---") {
292 return (DocumentFrontmatter::Absent, None);
293 }
294 let closing_line =
295 (2..=lines.line_count()).find(|line| lines.line_text(source, *line) == Some("---"));
296 let Some(closing_line) = closing_line else {
297 let location = FrontmatterLocation {
298 range: text_range(0, source.len()),
299 start_line: 1,
300 end_line: lines.line_count() as u64,
301 };
302 return (
303 DocumentFrontmatter::Invalid {
304 location,
305 message: "frontmatter opening delimiter has no closing `---` line".into(),
306 },
307 Some(0..source.len()),
308 );
309 };
310 let body_start = lines.line_start(2);
311 let body_end = lines.line_start(closing_line);
312 let block_end = lines.line_terminator_end(closing_line, source.len());
313 let range = 0..block_end;
314 let location = FrontmatterLocation {
315 range: text_range(range.start, range.end),
316 start_line: 1,
317 end_line: closing_line as u64,
318 };
319 let body = source.get(body_start..body_end).unwrap_or_default();
320 let (body, mark) = match body.strip_prefix('\u{feff}') {
329 Some(body) => (body, 1),
330 None => (body, 0),
331 };
332 let frontmatter = match exact_frontmatter_mapping(body, mark) {
333 Ok((value, positions)) => DocumentFrontmatter::Mapping {
334 value,
335 location,
336 anchors: document_frontmatter_anchors(source, lines, &location, positions, mark),
337 },
338 Err(message) => DocumentFrontmatter::Invalid { location, message },
339 };
340 (frontmatter, Some(range))
341}
342
343type BodyAnchors = Vec<(String, BodyPosition)>;
347
348#[derive(Clone, Copy, Debug, PartialEq, Eq)]
349struct BodyPosition {
350 line: usize,
351 column: usize,
352}
353
354fn body_position(span: &Span) -> BodyPosition {
362 BodyPosition {
363 line: span.start.line(),
364 column: span.start.col() + 1,
365 }
366}
367
368fn document_frontmatter_anchors(
390 source: &str,
391 lines: &LineIndex,
392 location: &FrontmatterLocation,
393 mut positions: BodyAnchors,
394 mark: usize,
395) -> FrontmatterAnchors {
396 positions.sort_unstable_by_key(|(_, position)| (position.line, position.column));
397 let mut anchors = BTreeMap::new();
398 let mut cursor = LineCursor::default();
399 for (pointer, position) in positions {
400 let Some(line) = position.line.checked_add(1) else {
401 continue;
402 };
403 if line < 2 || line as u64 >= location.end_line {
405 continue;
406 }
407 if cursor.line != line {
408 let Some(text) = lines.line_text(source, line) else {
409 continue;
410 };
411 cursor = LineCursor::new(line, text);
412 }
413 let shift = if position.line == 1 { mark } else { 0 };
414 let Some(column) = cursor.byte_column(position.column + shift) else {
415 continue;
416 };
417 anchors.insert(
418 pointer,
419 FrontmatterAnchor {
420 line: line as u64,
421 column,
422 },
423 );
424 }
425 FrontmatterAnchors(anchors)
426}
427
428#[derive(Default)]
434struct LineCursor<'a> {
435 line: usize,
437 rest: &'a str,
439 column: usize,
441 byte: usize,
443}
444
445impl<'a> LineCursor<'a> {
446 fn new(line: usize, text: &'a str) -> Self {
447 Self {
448 line,
449 rest: text,
450 column: 1,
451 byte: 0,
452 }
453 }
454
455 fn byte_column(&mut self, character_column: usize) -> Option<u64> {
456 if character_column < self.column {
457 return None;
458 }
459 while self.column < character_column {
460 let character = self.rest.chars().next()?;
461 self.rest = self.rest.get(character.len_utf8()..)?;
462 self.byte += character.len_utf8();
463 self.column += 1;
464 }
465 Some(self.byte as u64 + 1)
466 }
467}
468
469pub(crate) const MAX_YAML_DEPTH: usize = 128;
488
489#[derive(Clone, Copy, Debug, PartialEq, Eq)]
495pub(crate) struct YamlLimitExceeded;
496
497fn push_pointer_token(pointer: &mut String, token: &str) {
499 pointer.push('/');
500 for character in token.chars() {
501 match character {
502 '~' => pointer.push_str("~0"),
503 '/' => pointer.push_str("~1"),
504 _ => pointer.push(character),
505 }
506 }
507}
508
509#[derive(Clone, Debug, PartialEq, Eq)]
516pub(crate) enum YamlValueError {
517 TaggedNull,
519 TaggedBool,
521 TaggedInt,
523 TaggedFloat,
525 ScalarTag,
527 ContainerTag(&'static str),
529 NonFinite,
531 Unrepresentable { lexeme: String, error: String },
533}
534
535fn json_number(source: &str) -> Result<serde_json::Value, YamlValueError> {
536 serde_json::from_str(source).map_err(|error| YamlValueError::Unrepresentable {
537 lexeme: source.to_owned(),
538 error: error.to_string(),
539 })
540}
541
542#[derive(Clone, Copy, Debug, PartialEq, Eq)]
543enum JsonNumberKind {
544 Integer,
545 Float,
546}
547
548fn json_number_preserving_lexeme(
549 source: &str,
550 canonical: &str,
551 expected_kind: JsonNumberKind,
552) -> Result<serde_json::Value, YamlValueError> {
553 let source_kind = if source
556 .bytes()
557 .any(|byte| matches!(byte, b'.' | b'e' | b'E'))
558 {
559 JsonNumberKind::Float
560 } else {
561 JsonNumberKind::Integer
562 };
563 if source_kind == expected_kind && serde_json::from_str::<serde_json::Number>(source).is_ok() {
564 return Ok(serde_json::Value::Number(
568 serde_json::Number::from_string_unchecked(source.to_owned()),
569 ));
570 }
571 json_number(canonical)
572}
573
574#[derive(Clone, Debug, PartialEq, Eq, Hash)]
585enum ExactYamlNode {
586 Scalar(ExactYamlScalar),
587 Sequence {
588 tag: Option<YamlTag>,
589 values: Vec<SpannedYamlNode>,
590 },
591 Mapping {
592 tag: Option<YamlTag>,
593 entries: Vec<(SpannedYamlNode, SpannedYamlNode)>,
594 },
595}
596
597#[derive(Clone, Debug)]
606struct SpannedYamlNode {
607 node: ExactYamlNode,
608 position: BodyPosition,
610 expanded: bool,
617}
618
619impl PartialEq for SpannedYamlNode {
620 fn eq(&self, other: &Self) -> bool {
621 self.node == other.node
622 }
623}
624
625impl Eq for SpannedYamlNode {}
626
627impl std::hash::Hash for SpannedYamlNode {
628 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
629 self.node.hash(state);
630 }
631}
632
633#[derive(Clone, Debug, PartialEq, Eq, Hash)]
640pub(crate) struct ExactYamlScalar {
641 pub(crate) value: String,
642 pub(crate) style: ScalarStyle,
643 pub(crate) tag: Option<YamlTag>,
644}
645
646fn exact_yaml_key_digest(key: &SpannedYamlNode) -> u64 {
655 let mut hasher = std::hash::DefaultHasher::new();
656 std::hash::Hash::hash(key, &mut hasher);
657 std::hash::Hasher::finish(&hasher)
658}
659
660#[cfg(test)]
661thread_local! {
662 static KEY_COMPARISONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
671}
672
673fn exact_yaml_keys_equal(left: &SpannedYamlNode, right: &SpannedYamlNode) -> bool {
679 #[cfg(test)]
680 KEY_COMPARISONS.with(|made| made.set(made.get() + 1));
681 left == right
682}
683
684fn exact_frontmatter_mapping(
685 source: &str,
686 mark: usize,
687) -> Result<(serde_json::Map<String, serde_json::Value>, BodyAnchors), String> {
688 let tree = parse_exact_yaml(source, mark)?;
689 let mut pointer = String::new();
690 let mut anchors = BodyAnchors::new();
691 let value = exact_yaml_to_json(tree, &mut pointer, &mut anchors, None)?;
692 let serde_json::Value::Object(mapping) = value else {
693 return Err("frontmatter must be a YAML mapping".into());
694 };
695 Ok((mapping, anchors))
696}
697
698pub(crate) const EXACT_YAML_NODES_PER_EVENT: usize = 100;
708
709#[derive(Debug, Default)]
723pub(crate) struct ExactYamlBudget {
724 pub(crate) events: usize,
725 pub(crate) nodes: usize,
726}
727
728impl ExactYamlBudget {
729 pub(crate) fn spend(&mut self, nodes: usize) -> Result<(), YamlLimitExceeded> {
734 self.nodes = self.nodes.saturating_add(nodes);
735 if self.nodes > self.events.saturating_mul(EXACT_YAML_NODES_PER_EVENT) {
736 return Err(YamlLimitExceeded);
737 }
738 Ok(())
739 }
740}
741
742#[derive(Debug)]
750struct ExactYamlSubtree {
751 node: SpannedYamlNode,
752 depth: usize,
753}
754
755#[derive(Debug)]
766struct AnchoredYamlNode {
767 node: SpannedYamlNode,
768 nodes: usize,
769 depth: usize,
770}
771
772fn frontmatter_alias_error(YamlLimitExceeded: YamlLimitExceeded) -> String {
774 "frontmatter expands YAML aliases beyond its size limit".into()
775}
776
777fn frontmatter_depth_error(YamlLimitExceeded: YamlLimitExceeded) -> String {
779 "frontmatter nests YAML beyond its depth limit".into()
780}
781
782fn frontmatter_value_error(error: YamlValueError) -> String {
784 match error {
785 YamlValueError::TaggedNull => {
786 "frontmatter contains an invalid explicitly tagged null".into()
787 }
788 YamlValueError::TaggedBool => {
789 "frontmatter contains an invalid explicitly tagged boolean".into()
790 }
791 YamlValueError::TaggedInt => {
792 "frontmatter contains an invalid explicitly tagged integer".into()
793 }
794 YamlValueError::TaggedFloat => {
795 "frontmatter contains an invalid explicitly tagged float".into()
796 }
797 YamlValueError::ScalarTag => "frontmatter contains an invalid tag for a YAML scalar".into(),
798 YamlValueError::ContainerTag(expected) => {
799 format!("frontmatter contains an invalid tag for a YAML {expected}")
800 }
801 YamlValueError::NonFinite => "frontmatter contains a non-finite number".into(),
802 YamlValueError::Unrepresentable { lexeme, error } => {
803 format!("frontmatter number `{lexeme}` is not representable: {error}")
804 }
805 }
806}
807
808struct ExactYamlReader<'source> {
818 parser: ExactParser<'source, StrInput<'source>>,
819 anchors: BTreeMap<usize, AnchoredYamlNode>,
820 budget: ExactYamlBudget,
821 mark: usize,
824}
825
826impl<'source> ExactYamlReader<'source> {
827 fn new(source: &'source str, mark: usize) -> Self {
828 Self {
829 parser: ExactParser::new_from_str(source),
830 anchors: BTreeMap::new(),
831 budget: ExactYamlBudget::default(),
832 mark,
833 }
834 }
835
836 fn next_event(&mut self) -> Result<(ExactEvent<'source>, Span), String> {
842 self.budget.events += 1;
843 match self.parser.next_event() {
844 Some(Ok(read)) => Ok(read),
845 Some(Err(error)) => Err(self.syntax_error(&error)),
846 None => Err("frontmatter contains an unexpected YAML document boundary".into()),
847 }
848 }
849
850 fn spelled_position(&self, marker: &Marker) -> (usize, usize) {
857 (
858 marker.index() + self.mark,
859 marker.col() + 1 + if marker.line() == 1 { self.mark } else { 0 },
860 )
861 }
862
863 fn syntax_error(&self, error: &ScanError) -> String {
870 let marker = error.marker();
871 let (index, column) = self.spelled_position(marker);
872 format!(
873 "invalid YAML frontmatter: {} at byte {index} line {} column {column}",
874 error.info(),
875 marker.line(),
876 )
877 }
878
879 fn second_document_error(&self, span: &Span) -> String {
885 let (index, column) = self.spelled_position(&span.start);
886 format!(
887 "frontmatter must be a single YAML document: \
888 a second one opens at byte {index} line {} column {column}",
889 span.start.line(),
890 )
891 }
892
893 fn expect_event(
895 &mut self,
896 expected: impl FnOnce(&ExactEvent<'source>) -> bool,
897 ) -> Result<(), String> {
898 let (event, _) = self.next_event()?;
899 if expected(&event) {
900 Ok(())
901 } else {
902 Err("frontmatter contains an unexpected YAML document boundary".into())
903 }
904 }
905
906 fn node(
915 &mut self,
916 event: ExactEvent<'source>,
917 span: Span,
918 depth: usize,
919 ) -> Result<ExactYamlSubtree, String> {
920 let spent = self.budget.nodes;
921 let position = body_position(&span);
925 let (node, anchor, reached, expanded) = match event {
926 ExactEvent::Scalar(value, style, anchor, tag) => {
927 self.budget.spend(1).map_err(frontmatter_alias_error)?;
928 (
929 ExactYamlNode::Scalar(ExactYamlScalar {
930 value: value.into_owned(),
931 style,
932 tag: tag.map(Cow::into_owned),
933 }),
934 anchor,
935 0,
936 false,
937 )
938 }
939 ExactEvent::SequenceStart(anchor, tag) => {
940 let depth = deeper_yaml_nesting(depth, 1).map_err(frontmatter_depth_error)?;
941 self.budget.spend(1).map_err(frontmatter_alias_error)?;
942 let mut values = Vec::new();
943 let mut inner = 0;
944 loop {
945 let (event, span) = self.next_event()?;
946 if matches!(event, ExactEvent::SequenceEnd) {
947 break;
948 }
949 let value = self.node(event, span, depth)?;
950 inner = inner.max(value.depth);
951 values.push(value.node);
952 }
953 (
954 ExactYamlNode::Sequence {
955 tag: tag.map(Cow::into_owned),
956 values,
957 },
958 anchor,
959 inner + 1,
960 false,
961 )
962 }
963 ExactEvent::MappingStart(anchor, tag) => {
964 let depth = deeper_yaml_nesting(depth, 1).map_err(frontmatter_depth_error)?;
965 self.budget.spend(1).map_err(frontmatter_alias_error)?;
966 let mut entries: Vec<(SpannedYamlNode, SpannedYamlNode)> = Vec::new();
967 let mut keys: BTreeMap<u64, Vec<usize>> = BTreeMap::new();
968 let mut inner = 0;
969 loop {
970 let (event, span) = self.next_event()?;
971 if matches!(event, ExactEvent::MappingEnd) {
972 break;
973 }
974 let key = self.node(event, span, depth)?;
975 let (event, span) = self.next_event()?;
976 let value = self.node(event, span, depth)?;
977 inner = inner.max(key.depth).max(value.depth);
978 let (key, value) = (key.node, value.node);
979 let digest = exact_yaml_key_digest(&key);
1000 let alike = keys.entry(digest).or_default();
1001 if alike
1002 .iter()
1003 .any(|&entry| exact_yaml_keys_equal(&entries[entry].0, &key))
1004 {
1005 return Err("frontmatter contains a duplicate mapping key".into());
1006 }
1007 alike.push(entries.len());
1008 entries.push((key, value));
1009 }
1010 (
1011 ExactYamlNode::Mapping {
1012 tag: tag.map(Cow::into_owned),
1013 entries,
1014 },
1015 anchor,
1016 inner + 1,
1017 false,
1018 )
1019 }
1020 ExactEvent::Alias(anchor) => {
1021 let anchored = self
1022 .anchors
1023 .get(&anchor)
1024 .ok_or("frontmatter contains an unresolved YAML alias")?;
1025 let reached = anchored.depth;
1031 deeper_yaml_nesting(depth, reached).map_err(frontmatter_depth_error)?;
1032 self.budget
1040 .spend(anchored.nodes)
1041 .map_err(frontmatter_alias_error)?;
1042 (anchored.node.node.clone(), 0, reached, true)
1049 }
1050 _ => return Err("frontmatter contains an unexpected YAML parser event".into()),
1051 };
1052 let node = SpannedYamlNode {
1053 node,
1054 position,
1055 expanded,
1056 };
1057 self.remember_anchor(anchor, &node, self.budget.nodes - spent, reached);
1058 Ok(ExactYamlSubtree {
1059 node,
1060 depth: reached,
1061 })
1062 }
1063
1064 fn remember_anchor(
1071 &mut self,
1072 anchor: usize,
1073 node: &SpannedYamlNode,
1074 nodes: usize,
1075 depth: usize,
1076 ) {
1077 if anchor != 0 {
1078 self.anchors.insert(
1079 anchor,
1080 AnchoredYamlNode {
1081 node: node.clone(),
1082 nodes,
1083 depth,
1084 },
1085 );
1086 }
1087 }
1088}
1089
1090pub(crate) fn deeper_yaml_nesting(depth: usize, levels: usize) -> Result<usize, YamlLimitExceeded> {
1101 let depth = depth.saturating_add(levels);
1102 if depth > MAX_YAML_DEPTH {
1103 return Err(YamlLimitExceeded);
1104 }
1105 Ok(depth)
1106}
1107
1108fn parse_exact_yaml(source: &str, mark: usize) -> Result<SpannedYamlNode, String> {
1124 let mut reader = ExactYamlReader::new(source, mark);
1125 reader.expect_event(|event| matches!(event, ExactEvent::StreamStart))?;
1126 let (event, _) = reader.next_event()?;
1127 if matches!(event, ExactEvent::StreamEnd) {
1128 return Err("frontmatter must be a YAML mapping".into());
1129 }
1130 if !matches!(event, ExactEvent::DocumentStart(_)) {
1136 return Err("frontmatter contains an unexpected YAML document boundary".into());
1137 }
1138 let (event, span) = reader.next_event()?;
1139 let value = reader.node(event, span, 0)?.node;
1140 reader.expect_event(|event| matches!(event, ExactEvent::DocumentEnd))?;
1141 match reader.next_event() {
1142 Ok((ExactEvent::StreamEnd, _)) => Ok(value),
1143 Ok((ExactEvent::DocumentStart(_), span)) => Err(reader.second_document_error(&span)),
1144 _ => Err("frontmatter must be a single YAML document".into()),
1149 }
1150}
1151
1152fn exact_yaml_to_json(
1158 value: SpannedYamlNode,
1159 pointer: &mut String,
1160 anchors: &mut BodyAnchors,
1161 expansion: Option<BodyPosition>,
1162) -> Result<serde_json::Value, String> {
1163 let expansion = expansion.or_else(|| value.expanded.then_some(value.position));
1164 match value.node {
1165 ExactYamlNode::Scalar(scalar) => {
1166 exact_yaml_scalar_to_json(scalar).map_err(frontmatter_value_error)
1167 }
1168 ExactYamlNode::Sequence { tag, values } => {
1169 validate_yaml_container_tag(tag.as_ref(), "seq").map_err(frontmatter_value_error)?;
1170 let mut converted = Vec::with_capacity(values.len());
1171 for (index, value) in values.into_iter().enumerate() {
1172 let restore = pointer.len();
1173 pointer.push('/');
1175 pointer.push_str(&index.to_string());
1176 record_body_anchor(anchors, pointer, entry_anchor(&value, expansion));
1178 converted.push(exact_yaml_to_json(value, pointer, anchors, expansion)?);
1179 pointer.truncate(restore);
1180 }
1181 Ok(serde_json::Value::Array(converted))
1182 }
1183 ExactYamlNode::Mapping { tag, entries } => {
1184 validate_yaml_container_tag(tag.as_ref(), "map").map_err(frontmatter_value_error)?;
1185 exact_yaml_mapping_to_json(entries, pointer, anchors, expansion)
1186 }
1187 }
1188}
1189
1190fn exact_yaml_mapping_to_json(
1191 mapping: Vec<(SpannedYamlNode, SpannedYamlNode)>,
1192 pointer: &mut String,
1193 anchors: &mut BodyAnchors,
1194 expansion: Option<BodyPosition>,
1195) -> Result<serde_json::Value, String> {
1196 let mut object = serde_json::Map::new();
1197 for (key, value) in mapping {
1198 let position = entry_anchor(&key, expansion);
1199 let ExactYamlNode::Scalar(key) = key.node else {
1200 return Err("frontmatter mapping keys must be strings".into());
1201 };
1202 let serde_json::Value::String(key) =
1203 exact_yaml_scalar_to_json(key).map_err(frontmatter_value_error)?
1204 else {
1205 return Err("frontmatter mapping keys must be strings".into());
1206 };
1207 let restore = pointer.len();
1208 push_pointer_token(pointer, &key);
1209 record_body_anchor(anchors, pointer, position);
1211 let converted = exact_yaml_to_json(value, pointer, anchors, expansion)?;
1212 pointer.truncate(restore);
1213 if object.insert(key, converted).is_some() {
1214 return Err("frontmatter contains a duplicate mapping key".into());
1215 }
1216 }
1217 Ok(serde_json::Value::Object(object))
1218}
1219
1220fn entry_anchor(entry: &SpannedYamlNode, expansion: Option<BodyPosition>) -> Option<BodyPosition> {
1228 if let Some(site) = expansion {
1229 return Some(site);
1230 }
1231 if !entry.expanded {
1232 if let ExactYamlNode::Scalar(scalar) = &entry.node {
1233 if is_textless(scalar) {
1234 return None;
1235 }
1236 }
1237 }
1238 Some(entry.position)
1239}
1240
1241fn is_textless(scalar: &ExactYamlScalar) -> bool {
1262 matches!(scalar.style, ScalarStyle::Literal | ScalarStyle::Folded)
1263 && scalar.value.bytes().all(|byte| byte == b'\n')
1264}
1265
1266fn record_body_anchor(anchors: &mut BodyAnchors, pointer: &str, position: Option<BodyPosition>) {
1267 if let Some(position) = position {
1268 anchors.push((pointer.to_owned(), position));
1269 }
1270}
1271
1272pub(crate) fn validate_yaml_container_tag(
1274 tag: Option<&YamlTag>,
1275 expected: &'static str,
1276) -> Result<(), YamlValueError> {
1277 if standard_yaml_tag(tag).is_none_or(|tag| tag == expected) {
1278 Ok(())
1279 } else {
1280 Err(YamlValueError::ContainerTag(expected))
1281 }
1282}
1283
1284pub(crate) fn exact_yaml_scalar_to_json(
1289 scalar: ExactYamlScalar,
1290) -> Result<serde_json::Value, YamlValueError> {
1291 let standard_tag = standard_yaml_tag(scalar.tag.as_ref());
1292 match standard_tag {
1293 Some("str") => Ok(serde_json::Value::String(scalar.value)),
1294 Some("null") => match scalar.value.as_str() {
1295 "null" | "Null" | "NULL" | "~" => Ok(serde_json::Value::Null),
1296 _ => Err(YamlValueError::TaggedNull),
1297 },
1298 Some("bool") => match scalar.value.as_str() {
1299 "true" | "True" | "TRUE" => Ok(serde_json::Value::Bool(true)),
1300 "false" | "False" | "FALSE" => Ok(serde_json::Value::Bool(false)),
1301 _ => Err(YamlValueError::TaggedBool),
1302 },
1303 Some("int") => exact_yaml_integer(&scalar.value),
1304 Some("float") => exact_yaml_float(&scalar.value),
1305 Some("seq" | "map") => Err(YamlValueError::ScalarTag),
1306 Some(_) => Ok(serde_json::Value::String(scalar.value)),
1307 None if scalar.style != ScalarStyle::Plain => Ok(serde_json::Value::String(scalar.value)),
1308 None => plain_scalar_to_json(&scalar.value),
1309 }
1310}
1311
1312fn standard_yaml_tag(tag: Option<&YamlTag>) -> Option<&str> {
1313 tag.and_then(|tag| tag.is_yaml_core_schema().then_some(tag.suffix.as_str()))
1314}
1315
1316fn exact_yaml_integer(source: &str) -> Result<serde_json::Value, YamlValueError> {
1317 let canonical = canonical_tagged_yaml_integer(source).ok_or(YamlValueError::TaggedInt)?;
1318 json_number_preserving_lexeme(source, &canonical, JsonNumberKind::Integer)
1319}
1320
1321fn canonical_tagged_yaml_integer(source: &str) -> Option<String> {
1322 let (negative, unsigned) = if let Some(unsigned) = source.strip_prefix('-') {
1323 (true, unsigned)
1324 } else {
1325 (false, source.strip_prefix('+').unwrap_or(source))
1326 };
1327 if unsigned.starts_with(['+', '-']) {
1328 return None;
1329 }
1330 let (base, digits) = if let Some(digits) = unsigned.strip_prefix("0x") {
1331 (16, digits)
1332 } else if let Some(digits) = unsigned.strip_prefix("0o") {
1333 (8, digits)
1334 } else if let Some(digits) = unsigned.strip_prefix("0b") {
1335 (2, digits)
1336 } else {
1337 if unsigned.len() > 1 && unsigned.starts_with('0') {
1338 return None;
1339 }
1340 (10, unsigned)
1341 };
1342 if digits.is_empty() {
1343 return None;
1344 }
1345 let value = BigUint::parse_bytes(digits.as_bytes(), base)?;
1346 if value == BigUint::from(0_u8) {
1347 Some("0".into())
1348 } else {
1349 Some(format!("{}{value}", if negative { "-" } else { "" }))
1350 }
1351}
1352
1353fn exact_yaml_float(source: &str) -> Result<serde_json::Value, YamlValueError> {
1354 if let Some(canonical) = crate::loader::canonical_float(source) {
1355 if matches!(canonical.as_str(), "inf" | "-inf" | "nan") {
1356 return Err(YamlValueError::NonFinite);
1357 }
1358 return json_number_preserving_lexeme(source, &canonical, JsonNumberKind::Float);
1359 }
1360 let unsigned = source.strip_prefix(['-', '+']).unwrap_or(source);
1361 let crate::FrontmatterScalar::Integer(value) = crate::loader::parse_frontmatter_scalar(source)
1362 else {
1363 return Err(YamlValueError::TaggedFloat);
1364 };
1365 if unsigned.is_empty() || !unsigned.bytes().all(|byte| byte.is_ascii_digit()) {
1366 return Err(YamlValueError::TaggedFloat);
1367 }
1368 json_number(&format!("{}e0", value.0))
1369}
1370
1371fn plain_scalar_to_json(source: &str) -> Result<serde_json::Value, YamlValueError> {
1373 match crate::loader::parse_frontmatter_scalar(source) {
1374 crate::FrontmatterScalar::Null => Ok(serde_json::Value::Null),
1375 crate::FrontmatterScalar::Boolean(value) => Ok(serde_json::Value::Bool(value)),
1376 crate::FrontmatterScalar::Integer(value) => {
1377 json_number_preserving_lexeme(source, &value.0, JsonNumberKind::Integer)
1378 }
1379 crate::FrontmatterScalar::Float(value) => {
1380 if matches!(value.0.as_str(), "inf" | "-inf" | "nan") {
1381 Err(YamlValueError::NonFinite)
1382 } else {
1383 json_number_preserving_lexeme(source, &value.0, JsonNumberKind::Float)
1384 }
1385 }
1386 crate::FrontmatterScalar::String(value) => Ok(serde_json::Value::String(value)),
1387 }
1388}
1389
1390fn mask_source_range(source: &str, range: std::ops::Range<usize>) -> String {
1391 let bytes = source
1392 .bytes()
1393 .enumerate()
1394 .map(|(index, byte)| {
1395 if range.contains(&index) && !matches!(byte, b'\r' | b'\n') {
1396 b' '
1397 } else {
1398 byte
1399 }
1400 })
1401 .collect();
1402 match String::from_utf8(bytes) {
1403 Ok(masked) => masked,
1404 Err(_) => source.to_owned(),
1407 }
1408}
1409
1410fn normalize_bare_cr(source: &str) -> Cow<'_, str> {
1411 let has_bare_cr =
1412 source.as_bytes().iter().enumerate().any(|(index, byte)| {
1413 *byte == b'\r' && source.as_bytes().get(index + 1) != Some(&b'\n')
1414 });
1415 if !has_bare_cr {
1416 return Cow::Borrowed(source);
1417 }
1418
1419 Cow::Owned(
1420 source
1421 .char_indices()
1422 .map(|(index, character)| {
1423 if character == '\r' && source.as_bytes().get(index + 1) != Some(&b'\n') {
1424 '\n'
1425 } else {
1426 character
1427 }
1428 })
1429 .collect(),
1430 )
1431}
1432
1433struct HeadingBuilder {
1434 level: HeaderLevel,
1435 range: std::ops::Range<usize>,
1436 diagnostic_text: String,
1437}
1438
1439impl HeadingBuilder {
1440 fn new(level: HeadingLevel, range: std::ops::Range<usize>) -> Self {
1441 Self {
1442 level: convert_level(level),
1443 range,
1444 diagnostic_text: String::new(),
1445 }
1446 }
1447
1448 fn push_visible(&mut self, text: &str) {
1449 self.diagnostic_text.push_str(text);
1450 }
1451
1452 fn finish(
1453 self,
1454 source: &str,
1455 options: MarkdownOptions,
1456 lines: &LineIndex,
1457 line_suppressions: &BTreeMap<usize, Suppressions>,
1458 ) -> Heading {
1459 let safe_range = clamp_range(self.range, source.len());
1460 let line = lines.line_number(safe_range.start);
1461 let line_start = lines.line_start(line);
1462 let line_end = lines.line_end(line, source.len());
1463 let source_block = source.get(safe_range.clone()).unwrap_or_default();
1464 let source_text = extract_heading_source(source_block);
1465 let text = if options.strip_inline_markup {
1466 self.diagnostic_text.clone()
1467 } else {
1468 process_inline_text(&source_text)
1469 };
1470 let suppressions = line
1471 .checked_sub(1)
1472 .and_then(|prior| line_suppressions.get(&prior))
1473 .cloned()
1474 .unwrap_or_default();
1475
1476 Heading {
1477 level: self.level,
1478 text,
1479 diagnostic_text: self.diagnostic_text,
1480 source_text,
1481 location: HeadingLocation {
1482 range: text_range(safe_range.start, safe_range.end),
1483 line_range: text_range(line_start, line_end),
1484 line: line as u64,
1485 column: byte_column(line_start, safe_range.start),
1486 },
1487 suppressions,
1488 }
1489 }
1490}
1491
1492fn is_eligible_heading(
1493 source: &str,
1494 range: &std::ops::Range<usize>,
1495 event_level: HeadingLevel,
1496 lines: &LineIndex,
1497) -> bool {
1498 let safe_range = clamp_range(range.clone(), source.len());
1499 let first_line = lines.line_number(safe_range.start);
1500 let line_start = lines.line_start(first_line);
1501 let Some(prefix) = source.get(line_start..safe_range.start) else {
1502 return false;
1503 };
1504 if prefix.len() > 3 || !prefix.bytes().all(|byte| byte == b' ') {
1505 return false;
1506 }
1507
1508 let Some(first_text) = lines.line_text(source, first_line) else {
1509 return false;
1510 };
1511 if let Some(level) = physical_atx_level(first_text) {
1512 return level == convert_level(event_level);
1513 }
1514
1515 if !matches!(event_level, HeadingLevel::H1 | HeadingLevel::H2) {
1516 return false;
1517 }
1518 let last_offset = safe_range
1519 .end
1520 .checked_sub(1)
1521 .unwrap_or(safe_range.start)
1522 .max(safe_range.start);
1523 let last_line = lines.line_number(last_offset.min(source.len()));
1524 lines
1525 .line_text(source, last_line)
1526 .is_some_and(|line| setext_level(line) == Some(convert_level(event_level)))
1527}
1528
1529fn physical_atx_level(line: &str) -> Option<HeaderLevel> {
1530 let bytes = line.as_bytes();
1531 let indent = bytes.iter().take_while(|byte| **byte == b' ').count();
1532 if indent > 3 {
1533 return None;
1534 }
1535 let hashes = bytes
1536 .get(indent..)?
1537 .iter()
1538 .take_while(|byte| **byte == b'#')
1539 .count();
1540 if !(1..=6).contains(&hashes) {
1541 return None;
1542 }
1543 let after = indent + hashes;
1544 if bytes.get(after).is_some_and(|byte| *byte != b' ') {
1545 return None;
1546 }
1547 u8::try_from(hashes)
1548 .ok()
1549 .and_then(|level| HeaderLevel::try_from(level).ok())
1550}
1551
1552fn convert_level(level: HeadingLevel) -> HeaderLevel {
1553 match level {
1554 HeadingLevel::H1 => HeaderLevel::H1,
1555 HeadingLevel::H2 => HeaderLevel::H2,
1556 HeadingLevel::H3 => HeaderLevel::H3,
1557 HeadingLevel::H4 => HeaderLevel::H4,
1558 HeadingLevel::H5 => HeaderLevel::H5,
1559 HeadingLevel::H6 => HeaderLevel::H6,
1560 }
1561}
1562
1563fn clamp_range(range: std::ops::Range<usize>, source_len: usize) -> std::ops::Range<usize> {
1564 range.start.min(source_len)..range.end.min(source_len).max(range.start.min(source_len))
1565}
1566
1567fn text_range(start: usize, end: usize) -> TextRange {
1568 TextRange {
1569 start: ByteOffset(start),
1570 end: ByteOffset(end),
1571 }
1572}
1573
1574fn byte_column(line_start: usize, offset: usize) -> u64 {
1575 (offset - line_start + 1) as u64
1576}
1577
1578fn extract_heading_source(block: &str) -> String {
1579 let mut lines = physical_lines(block);
1580 let first_line = lines.first().copied().unwrap_or_default();
1581 let trimmed_indent = first_line.trim_start_matches(' ');
1582 let hash_count = trimmed_indent
1583 .bytes()
1584 .take_while(|byte| *byte == b'#')
1585 .count();
1586
1587 if (1..=6).contains(&hash_count)
1588 && trimmed_indent
1589 .as_bytes()
1590 .get(hash_count)
1591 .is_none_or(|byte| *byte == b' ')
1592 {
1593 return trimmed_indent
1594 .get(hash_count..)
1595 .map(strip_atx_closing_hashes)
1596 .unwrap_or_default()
1597 .to_owned();
1598 }
1599
1600 if lines.last().is_some_and(|line| is_setext_underline(line)) {
1601 lines.pop();
1602 }
1603 lines.join("\n").trim().to_owned()
1604}
1605
1606fn strip_atx_closing_hashes(content: &str) -> &str {
1607 let content = content.trim_end();
1608 let without_hashes = content.trim_end_matches('#');
1609 if without_hashes.len() != content.len()
1610 && without_hashes
1611 .as_bytes()
1612 .last()
1613 .is_some_and(|byte| *byte == b' ')
1614 {
1615 without_hashes.trim()
1616 } else {
1617 content.trim()
1618 }
1619}
1620
1621fn is_setext_underline(line: &str) -> bool {
1622 setext_level(line).is_some()
1623}
1624
1625fn setext_level(line: &str) -> Option<HeaderLevel> {
1626 let bytes = line.as_bytes();
1627 let indent = bytes.iter().take_while(|byte| **byte == b' ').count();
1628 if indent > 3 {
1629 return None;
1630 }
1631 let marker = bytes.get(indent).copied()?;
1632 let level = match marker {
1633 b'=' => HeaderLevel::H1,
1634 b'-' => HeaderLevel::H2,
1635 _ => return None,
1636 };
1637 let marker_end = bytes
1638 .get(indent..)?
1639 .iter()
1640 .take_while(|byte| **byte == marker)
1641 .count()
1642 + indent;
1643 if bytes
1644 .get(marker_end..)
1645 .is_some_and(|trailing| !trailing.iter().all(|byte| matches!(byte, b' ' | b'\t')))
1646 {
1647 return None;
1648 }
1649 Some(level)
1650}
1651
1652fn physical_lines(source: &str) -> Vec<&str> {
1653 line_ranges(source)
1654 .into_iter()
1655 .filter(|line| line.start < source.len())
1656 .filter_map(|line| source.get(line.start..line.end))
1657 .collect()
1658}
1659
1660fn process_inline_text(source: &str) -> String {
1661 let mut replacements = Vec::new();
1662 for (event, range) in Parser::new_ext(source, CommonMarkOptions::empty()).into_offset_iter() {
1663 if let Event::Text(text) = event {
1664 let range = expand_escaped_punctuation(source, range, &text);
1665 if source
1666 .get(range.clone())
1667 .is_some_and(|raw| raw != text.as_ref())
1668 {
1669 replacements.push((range, text.into_string()));
1670 }
1671 }
1672 }
1673
1674 let mut output = String::with_capacity(source.len());
1675 let mut cursor = 0;
1676 for (range, replacement) in replacements {
1677 if range.start < cursor || range.end > source.len() {
1678 continue;
1679 }
1680 if let Some(unchanged) = source.get(cursor..range.start) {
1681 output.push_str(unchanged);
1682 }
1683 output.push_str(&replacement);
1684 cursor = range.end;
1685 }
1686 if let Some(remainder) = source.get(cursor..) {
1687 output.push_str(remainder);
1688 }
1689 output
1690}
1691
1692fn expand_escaped_punctuation(
1693 source: &str,
1694 range: std::ops::Range<usize>,
1695 text: &str,
1696) -> std::ops::Range<usize> {
1697 let escaped = text
1698 .as_bytes()
1699 .first()
1700 .is_some_and(u8::is_ascii_punctuation)
1701 && range
1702 .start
1703 .checked_sub(1)
1704 .and_then(|index| source.as_bytes().get(index))
1705 .is_some_and(|byte| *byte == b'\\');
1706 if escaped {
1707 range.start - 1..range.end
1708 } else {
1709 range
1710 }
1711}
1712
1713fn collect_suppressions(
1714 source: &str,
1715 html: &str,
1716 range: std::ops::Range<usize>,
1717 lines: &LineIndex,
1718 file: &mut Suppressions,
1719 per_line: &mut BTreeMap<usize, Suppressions>,
1720) {
1721 let safe_range = clamp_range(range, source.len());
1722 let raw_html = source.get(safe_range.clone()).unwrap_or(html);
1723 let base_offset = safe_range.start;
1724 let mut cursor = 0;
1725 while let Some(relative_start) = raw_html.get(cursor..).and_then(|raw| raw.find("<!--")) {
1726 let comment_start = cursor + relative_start;
1727 let body_start = comment_start + "<!--".len();
1728 let Some(relative_end) = raw_html.get(body_start..).and_then(|raw| raw.find("-->")) else {
1729 break;
1730 };
1731 let comment_end = body_start + relative_end + "-->".len();
1732 let Some(comment) = raw_html.get(comment_start..comment_end) else {
1733 break;
1734 };
1735 cursor = comment_end;
1736
1737 let Some((file_wide, suppressions)) = parse_suppression(comment) else {
1738 continue;
1739 };
1740 if file_wide {
1741 file.0.extend(suppressions.0);
1742 continue;
1743 }
1744
1745 let absolute_start = base_offset
1746 .checked_add(comment_start)
1747 .unwrap_or(source.len())
1748 .min(source.len());
1749 let line = lines.line_number(absolute_start);
1750 let is_entire_line = lines
1751 .line_text(source, line)
1752 .is_some_and(|line_text| line_text.trim() == comment);
1753 if is_entire_line {
1754 per_line.entry(line).or_default().0.extend(suppressions.0);
1755 }
1756 }
1757}
1758
1759fn parse_suppression(html: &str) -> Option<(bool, Suppressions)> {
1760 let comment = html
1761 .trim()
1762 .strip_prefix("<!--")?
1763 .strip_suffix("-->")?
1764 .trim();
1765 let (file_wide, ids) = if let Some(ids) = comment.strip_prefix("outlint-disable-file") {
1766 (true, ids)
1767 } else {
1768 (false, comment.strip_prefix("outlint-disable")?)
1769 };
1770 if !ids.starts_with(char::is_whitespace) {
1771 return None;
1772 }
1773
1774 let ids: BTreeSet<_> = ids
1775 .split(|character: char| character == ',' || character.is_whitespace())
1776 .filter(|id| !id.is_empty())
1777 .map(|id| SuppressedDiagnostic(id.to_owned()))
1778 .collect();
1779 if ids.is_empty() {
1780 None
1781 } else {
1782 Some((file_wide, Suppressions(ids)))
1783 }
1784}
1785
1786fn build_section_tree(headings: Vec<Heading>) -> Vec<Section> {
1787 let mut roots = Vec::new();
1788 let mut path = Vec::<usize>::new();
1789
1790 for heading in headings {
1791 while let Some(parent) = section_at_path(&roots, &path) {
1792 if parent.heading.level < heading.level {
1793 break;
1794 }
1795 path.pop();
1796 }
1797
1798 let Some(siblings) = children_at_path_mut(&mut roots, &path) else {
1799 continue;
1800 };
1801 siblings.push(Section {
1802 heading,
1803 children: Vec::new(),
1804 });
1805 path.push(siblings.len() - 1);
1806 }
1807
1808 roots
1809}
1810
1811fn section_at_path<'a>(roots: &'a [Section], path: &[usize]) -> Option<&'a Section> {
1812 let (first, rest) = path.split_first()?;
1813 let mut section = roots.get(*first)?;
1814 for index in rest {
1815 section = section.children.get(*index)?;
1816 }
1817 Some(section)
1818}
1819
1820fn children_at_path_mut<'a>(
1821 roots: &'a mut Vec<Section>,
1822 path: &[usize],
1823) -> Option<&'a mut Vec<Section>> {
1824 let Some((first, rest)) = path.split_first() else {
1825 return Some(roots);
1826 };
1827 let section = roots.get_mut(*first)?;
1828 children_at_path_mut(&mut section.children, rest)
1829}
1830
1831#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1832struct LineRange {
1833 start: usize,
1834 end: usize,
1835 terminator_end: usize,
1836}
1837
1838fn line_ranges(source: &str) -> Vec<LineRange> {
1839 let bytes = source.as_bytes();
1840 let mut lines = Vec::new();
1841 let mut start = 0;
1842 let mut index = 0;
1843 while index < bytes.len() {
1844 let terminator_end = match bytes[index] {
1845 b'\r' if bytes.get(index + 1) == Some(&b'\n') => index + 2,
1846 b'\r' | b'\n' => index + 1,
1847 _ => {
1848 index += 1;
1849 continue;
1850 }
1851 };
1852 lines.push(LineRange {
1853 start,
1854 end: index,
1855 terminator_end,
1856 });
1857 start = terminator_end;
1858 index = terminator_end;
1859 }
1860 lines.push(LineRange {
1861 start,
1862 end: source.len(),
1863 terminator_end: source.len(),
1864 });
1865 lines
1866}
1867
1868struct LineIndex {
1869 lines: Vec<LineRange>,
1870}
1871
1872impl LineIndex {
1873 fn new(source: &str) -> Self {
1874 Self {
1875 lines: line_ranges(source),
1876 }
1877 }
1878
1879 fn line_number(&self, offset: usize) -> usize {
1880 self.lines.partition_point(|line| line.start <= offset)
1881 }
1882
1883 fn line_start(&self, line: usize) -> usize {
1884 line.checked_sub(1)
1885 .and_then(|index| self.lines.get(index).map(|line| line.start))
1886 .unwrap_or_default()
1887 }
1888
1889 fn line_end(&self, line: usize, source_len: usize) -> usize {
1890 line.checked_sub(1)
1891 .and_then(|index| self.lines.get(index).map(|line| line.end))
1892 .unwrap_or(source_len)
1893 }
1894
1895 fn line_terminator_end(&self, line: usize, source_len: usize) -> usize {
1896 line.checked_sub(1)
1897 .and_then(|index| self.lines.get(index).map(|line| line.terminator_end))
1898 .unwrap_or(source_len)
1899 }
1900
1901 fn line_text<'a>(&self, source: &'a str, line: usize) -> Option<&'a str> {
1902 let start = self.line_start(line);
1903 let end = line
1904 .checked_sub(1)
1905 .and_then(|index| self.lines.get(index).map(|line| line.end))?;
1906 source.get(start..end)
1907 }
1908
1909 fn line_count(&self) -> usize {
1910 self.lines.len()
1911 }
1912}
1913
1914#[cfg(test)]
1915mod tests {
1916 use super::*;
1917 use proptest::prelude::*;
1918
1919 const NO_MARK: usize = 0;
1922
1923 fn headings(document: &Document) -> Vec<&Heading> {
1924 fn visit<'a>(sections: &'a [Section], output: &mut Vec<&'a Heading>) {
1925 for section in sections {
1926 output.push(§ion.heading);
1927 visit(§ion.children, output);
1928 }
1929 }
1930
1931 let mut output = Vec::new();
1932 visit(&document.sections, &mut output);
1933 output
1934 }
1935
1936 #[test]
1937 fn parses_atx_and_setext_headings_but_not_near_misses() {
1938 let source = concat!(
1939 "# one\n",
1940 " ## two ##\n",
1941 "####### no\n\n",
1942 " ### indented code\n\n",
1943 "no-space#\n\n",
1944 "setext one\n",
1945 "===\n",
1946 "setext two\n",
1947 "---\n",
1948 );
1949 let document = parse_markdown(source, MarkdownOptions::default());
1950 let actual: Vec<_> = headings(&document)
1951 .into_iter()
1952 .map(|heading| (heading.level, heading.text.as_str()))
1953 .collect();
1954
1955 assert_eq!(
1956 actual,
1957 [
1958 (HeaderLevel::H1, "one"),
1959 (HeaderLevel::H2, "two"),
1960 (HeaderLevel::H1, "setext one"),
1961 (HeaderLevel::H2, "setext two"),
1962 ]
1963 );
1964 }
1965
1966 #[test]
1967 fn accepts_only_top_level_physical_heading_lines() {
1968 let source = concat!(
1969 "> # quoted atx\n\n",
1970 "- # listed atx\n\n",
1971 "> quoted setext\n> ===\n\n",
1972 "- listed setext\n ---\n\n",
1973 "- containing item\n\n ### continued-list atx\n\n",
1974 "#\ttab is not the required literal space\n\n",
1975 " ## physical atx\n",
1976 "physical setext\n---\n",
1977 );
1978 let document = parse_markdown(source, MarkdownOptions::default());
1979 let actual: Vec<_> = headings(&document)
1980 .into_iter()
1981 .map(|heading| heading.text.as_str())
1982 .collect();
1983
1984 assert_eq!(actual, ["physical atx", "physical setext"]);
1985 }
1986
1987 #[test]
1988 fn ignores_headings_in_commonmark_fences() {
1989 let source = concat!(
1990 "~~~ rust\n# hidden\n~~~\n",
1991 " ```` language\n## also hidden\n``` not a close\n ````\n",
1992 "### visible\n",
1993 );
1994 let document = parse_markdown(source, MarkdownOptions::default());
1995 let actual: Vec<_> = headings(&document)
1996 .into_iter()
1997 .map(|heading| heading.text.as_str())
1998 .collect();
1999
2000 assert_eq!(actual, ["visible"]);
2001 }
2002
2003 #[test]
2004 fn applies_atx_closing_hash_rules() {
2005 let document = parse_markdown(
2006 "# text ###\n# text###\n# ###\n# text # tail\n",
2007 MarkdownOptions::default(),
2008 );
2009 let actual: Vec<_> = headings(&document)
2010 .into_iter()
2011 .map(|heading| (heading.text.as_str(), heading.source_text.as_str()))
2012 .collect();
2013
2014 assert_eq!(
2015 actual,
2016 [
2017 ("text", "text"),
2018 ("text###", "text###"),
2019 ("", ""),
2020 ("text # tail", "text # tail"),
2021 ]
2022 );
2023 }
2024
2025 #[test]
2026 fn strips_inline_markup_and_decodes_commonmark_text() {
2027 let source = "## **A&B** [link](target)  `code` <i>tag</i> \\*star\\*\n";
2028 let stripped = parse_markdown(source, MarkdownOptions::default());
2029 let preserved = parse_markdown(
2030 source,
2031 MarkdownOptions {
2032 strip_inline_markup: false,
2033 },
2034 );
2035
2036 let stripped_heading = &stripped.sections[0].heading;
2037 assert_eq!(stripped_heading.text, "A&B link alt code tag *star*");
2038 assert_eq!(stripped_heading.diagnostic_text, stripped_heading.text);
2039 assert_eq!(
2040 stripped_heading.source_text,
2041 "**A&B** [link](target)  `code` <i>tag</i> \\*star\\*"
2042 );
2043 assert_eq!(
2044 preserved.sections[0].heading.text,
2045 "**A&B** [link](target)  `code` <i>tag</i> *star*"
2046 );
2047 }
2048
2049 #[test]
2050 fn builds_tree_using_nearest_prior_lower_heading() {
2051 let document = parse_markdown(
2052 "# root\n### skipped\n#### child\n## sibling\n# next\n",
2053 MarkdownOptions::default(),
2054 );
2055
2056 assert_eq!(document.sections.len(), 2);
2057 assert_eq!(document.sections[0].children.len(), 2);
2058 assert_eq!(document.sections[0].children[0].heading.text, "skipped");
2059 assert_eq!(document.sections[0].children[0].children.len(), 1);
2060 assert_eq!(document.sections[0].children[1].heading.text, "sibling");
2061 }
2062
2063 #[test]
2064 fn records_byte_line_column_and_setext_extent() {
2065 let source = "å\n\n # atx\r\nsetext\n---\n";
2066 let document = parse_markdown(source, MarkdownOptions::default());
2067 let found = headings(&document);
2068
2069 assert_eq!(found[0].location.line, 3);
2070 assert_eq!(found[0].location.column, 4);
2071 assert_eq!(found[0].location.line_range, text_range(4, 12));
2072 assert_eq!(found[1].location.line, 4);
2073 assert_eq!(
2074 source.get(found[1].location.range.start.0..found[1].location.range.end.0),
2075 Some("setext\n---\n")
2076 );
2077 }
2078
2079 #[test]
2080 fn captures_header_and_file_suppressions() {
2081 let source = concat!(
2082 "<!-- outlint-disable-file missing-section, requires -->\n",
2083 "<!-- outlint-disable skipped-level, not-allowed -->\n",
2084 "## suppressed\n",
2085 "<!-- outlint-disable unexpected-section -->\n",
2086 "\n",
2087 "## not suppressed\n",
2088 );
2089 let document = parse_markdown(source, MarkdownOptions::default());
2090 let found = headings(&document);
2091
2092 assert!(document.file_suppressions.contains("missing-section"));
2093 assert!(document.file_suppressions.contains("requires"));
2094 assert!(found[0].suppressions.contains("skipped-level"));
2095 assert!(found[0].suppressions.contains("not-allowed"));
2096 assert!(found[1].suppressions.0.is_empty());
2097 }
2098
2099 #[test]
2100 fn finds_file_suppressions_nested_in_raw_html() {
2101 let source = concat!(
2102 "<div>\n",
2103 "before\n",
2104 "<!-- outlint-disable-file missing-section -->\n",
2105 "<!-- outlint-disable-file requires, ordered -->\n",
2106 "after\n",
2107 "</div>\n\n",
2108 "# heading\n",
2109 );
2110 let document = parse_markdown(source, MarkdownOptions::default());
2111
2112 assert!(document.file_suppressions.contains("missing-section"));
2113 assert!(document.file_suppressions.contains("requires"));
2114 assert!(document.file_suppressions.contains("ordered"));
2115 }
2116
2117 #[test]
2118 fn requires_header_suppression_to_occupy_its_whole_line() {
2119 let source = concat!(
2120 "prefix <!-- outlint-disable skipped-level -->\n",
2121 "# not suppressed\n",
2122 "<!-- outlint-disable skipped-level --> suffix\n",
2123 "# also not suppressed\n",
2124 );
2125 let document = parse_markdown(source, MarkdownOptions::default());
2126
2127 assert!(headings(&document)
2128 .iter()
2129 .all(|heading| !heading.suppressions.contains("skipped-level")));
2130 }
2131
2132 #[test]
2133 fn bare_cr_delimits_locations_and_suppression_lines() {
2134 let source = concat!(
2135 "<!-- outlint-disable skipped-level -->\r",
2136 " ## first\r",
2137 "setext\r",
2138 "---\r",
2139 );
2140 let document = parse_markdown(source, MarkdownOptions::default());
2141 let found = headings(&document);
2142
2143 assert_eq!(found.len(), 2);
2144 assert_eq!(found[0].location.line, 2);
2145 assert_eq!(found[0].location.column, 4);
2146 assert_eq!(found[0].location.line_range, text_range(39, 50));
2147 assert!(found[0].suppressions.contains("skipped-level"));
2148 assert_eq!(found[1].location.line, 3);
2149 assert_eq!(found[1].location.line_range, text_range(51, 57));
2150 }
2151
2152 #[test]
2153 fn line_index_treats_crlf_as_one_ending_and_cr_as_an_ending() {
2154 let source = "a\r\nb\rc\nd";
2155 let lines = LineIndex::new(source);
2156 let actual: Vec<_> = (1..=lines.line_count())
2157 .map(|line| lines.line_text(source, line))
2158 .collect();
2159
2160 assert_eq!(actual, [Some("a"), Some("b"), Some("c"), Some("d")]);
2161 assert_eq!(lines.line_number(3), 2);
2162 assert_eq!(lines.line_number(5), 3);
2163 assert_eq!(lines.line_number(7), 4);
2164 }
2165
2166 #[test]
2167 fn ignores_suppression_spelling_near_misses_and_code() {
2168 let source = concat!(
2169 "```html\n<!-- outlint-disable-file skipped-level -->\n```\n",
2170 "<!-- outlint-disable-filed not-allowed -->\n",
2171 "<!-- outlint-disable -->\n",
2172 "# heading\n",
2173 );
2174 let document = parse_markdown(source, MarkdownOptions::default());
2175
2176 assert!(document.file_suppressions.0.is_empty());
2177 assert!(document.sections[0].heading.suppressions.0.is_empty());
2178 }
2179
2180 #[test]
2181 fn parses_and_masks_yaml_frontmatter_before_heading_scanning() {
2182 let source = concat!(
2183 "---\n",
2184 "title: metadata, not a setext heading\n",
2185 "draft: false\n",
2186 "tags: [one, two]\n",
2187 "---\n",
2188 "# Document title\n",
2189 );
2190 let document = parse_markdown(source, MarkdownOptions::default());
2191
2192 let DocumentFrontmatter::Mapping {
2193 value, location, ..
2194 } = &document.frontmatter
2195 else {
2196 panic!("expected parsed frontmatter")
2197 };
2198 assert_eq!(value.get("draft"), Some(&serde_json::Value::Bool(false)));
2199 assert_eq!(location.start_line, 1);
2200 assert_eq!(location.end_line, 5);
2201 assert_eq!(headings(&document).len(), 1);
2202 assert_eq!(headings(&document)[0].diagnostic_text, "Document title");
2203 }
2204
2205 #[test]
2206 fn frontmatter_anchors_locate_entries_by_json_pointer() {
2207 let source = concat!(
2210 "---\n", "# a comment\n", "\n", "\n", "count: nope\n", "nested:\n", " inner: 1\n", "tags:\n", " - ok\n", " - 123\n", "flow: [\"ää\", 5]\n", "items:\n", " - key: 1\n", "flowseq: [{p: 1}, 5]\n", "weird/key~name: 1\n", "---\n", "# Title\n",
2227 );
2228 let document = parse_markdown(source, MarkdownOptions::default());
2229
2230 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2231 panic!("expected parsed frontmatter: {document:?}")
2232 };
2233 let anchor = |pointer: &str| {
2234 anchors
2235 .get(pointer)
2236 .map(|anchor| (anchor.line, anchor.column))
2237 };
2238
2239 assert_eq!(anchor("/count"), Some((5, 1)));
2242 assert_eq!(anchor("/nested"), Some((6, 1)));
2243 assert_eq!(anchor("/nested/inner"), Some((7, 3)));
2244 assert_eq!(anchor("/tags"), Some((8, 1)));
2245 assert_eq!(anchor("/tags/0"), Some((9, 5)));
2247 assert_eq!(anchor("/tags/1"), Some((10, 5)));
2248 assert_eq!(anchor("/flow/1"), Some((11, 16)));
2251 assert_eq!(anchor("/items/0"), Some((13, 5)));
2254 assert_eq!(anchor("/items/0/key"), Some((13, 5)));
2255 assert_eq!(anchor("/flowseq/0"), Some((14, 11)));
2258 assert_eq!(anchor("/flowseq/0/p"), Some((14, 12)));
2259 assert_eq!(anchor("/flowseq/1"), Some((14, 19)));
2260 assert_eq!(anchor("/weird~1key~0name"), Some((15, 1)));
2262 assert_eq!(anchor(""), None);
2264 assert_eq!(anchor("/absent"), None);
2265 }
2266
2267 #[test]
2268 fn frontmatter_anchors_convert_many_entries_on_one_line() {
2269 const ENTRIES: usize = 500;
2274 let mut line = String::from("ää: [");
2275 let mut columns = Vec::with_capacity(ENTRIES);
2276 for index in 0..ENTRIES {
2277 if index > 0 {
2278 line.push_str(", ");
2279 }
2280 columns.push(line.len() as u64 + 1);
2283 line.push_str(&index.to_string());
2284 }
2285 line.push(']');
2286 let source = format!("---\n{line}\n---\n# Title\n");
2287 let document = parse_markdown(&source, MarkdownOptions::default());
2288
2289 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2290 panic!("expected parsed frontmatter: {document:?}")
2291 };
2292 for (index, column) in columns.into_iter().enumerate() {
2293 assert_eq!(
2294 anchors.get(&format!("/ää/{index}")),
2295 Some(FrontmatterAnchor { line: 2, column }),
2296 "element {index} is misplaced"
2297 );
2298 }
2299 }
2300
2301 #[test]
2302 fn only_empty_block_scalars_take_no_anchor() {
2303 let source = concat!(
2315 "---\n", "gaps:\n", " -\n", " -\n", " - 3\n", "folded:\n", " - >-\n", " - 2\n", "literal:\n", " - |\n", " - 2\n", "kept:\n", " - |+\n", "\n", " - 2\n", "blanks:\n", " - |+\n", "\n", "\n", " - 2\n", "quoted:\n", " - \"\"\n", " - ''\n", " - 3\n", "spaced:\n", " - \" \"\n", " - \"\\r\"\n", " - \"\\t\"\n", "nulls:\n", " - null\n", " - ~\n", "written:\n", " - >-\n", " text\n", " - 2\n", "keyed:\n", " ? >-\n", " next: second\n", "trailing:\n", " - 1\n", " -\n", "---\n", "# Title\n",
2358 );
2359 let document = parse_markdown(source, MarkdownOptions::default());
2360
2361 let DocumentFrontmatter::Mapping { value, anchors, .. } = &document.frontmatter else {
2362 panic!("expected parsed frontmatter: {document:?}")
2363 };
2364 let anchor = |pointer: &str| {
2365 anchors
2366 .get(pointer)
2367 .map(|anchor| (anchor.line, anchor.column))
2368 };
2369
2370 assert_eq!(anchor("/gaps/0"), Some((3, 4)));
2373 assert_eq!(anchor("/gaps/1"), Some((4, 4)));
2374 assert_eq!(anchor("/gaps/2"), Some((5, 5)));
2375 assert_eq!(anchor("/folded/0"), None);
2379 assert_eq!(anchor("/folded/1"), Some((8, 5)));
2380 assert_eq!(anchor("/literal/0"), None);
2381 assert_eq!(anchor("/literal/1"), Some((11, 5)));
2382 assert_eq!(anchor("/kept/0"), None);
2385 assert_eq!(anchor("/kept/1"), Some((15, 5)));
2386 assert_eq!(
2387 value.get("kept"),
2388 Some(&serde_json::json!(["\n", 2])),
2389 "a kept blank line is still part of the value"
2390 );
2391 assert_eq!(anchor("/blanks/0"), None);
2397 assert_eq!(anchor("/blanks/1"), Some((20, 5)));
2398 assert_eq!(
2399 value.get("blanks"),
2400 Some(&serde_json::json!(["\n\n", 2])),
2401 "both kept blank lines are part of the value"
2402 );
2403 assert_eq!(anchor("/quoted/0"), Some((22, 5)));
2408 assert_eq!(anchor("/quoted/1"), Some((23, 5)));
2409 assert_eq!(anchor("/quoted/2"), Some((24, 5)));
2410 assert_eq!(
2411 value.get("quoted"),
2412 Some(&serde_json::json!(["", "", 3])),
2413 "quoted empties must stay strings"
2414 );
2415 assert_eq!(anchor("/spaced/0"), Some((26, 5)));
2421 assert_eq!(anchor("/spaced/1"), Some((27, 5)));
2422 assert_eq!(anchor("/spaced/2"), Some((28, 5)));
2423 assert_eq!(
2424 value.get("spaced"),
2425 Some(&serde_json::json!([" ", "\r", "\t"])),
2426 "each element holds the one whitespace character it spells"
2427 );
2428 assert_eq!(
2429 value.get("gaps"),
2430 Some(&serde_json::json!([null, null, 3])),
2431 "unwritten elements must stay null"
2432 );
2433 assert_eq!(anchor("/nulls/0"), Some((30, 5)));
2436 assert_eq!(anchor("/nulls/1"), Some((31, 5)));
2437 assert_eq!(
2438 value.get("nulls"),
2439 Some(&serde_json::json!([null, null])),
2440 "written nulls must parse as null"
2441 );
2442 assert_eq!(anchor("/written/0"), Some((34, 5)));
2445 assert_eq!(anchor("/written/1"), Some((35, 5)));
2446 assert_eq!(anchor("/keyed/"), None);
2450 assert_eq!(anchor("/keyed/next"), Some((38, 3)));
2451 assert_eq!(
2452 value.get("keyed"),
2453 Some(&serde_json::json!({"": null, "next": "second"})),
2454 "the explicit key parses to an empty-keyed member"
2455 );
2456 assert_eq!(anchor("/trailing/0"), Some((40, 5)));
2459 assert_eq!(anchor("/trailing/1"), Some((41, 4)));
2460
2461 let mut placed: Vec<_> = anchors
2463 .0
2464 .iter()
2465 .map(|(pointer, anchor)| (anchor.line, anchor.column, pointer.as_str()))
2466 .collect();
2467 placed.sort_unstable();
2468 for pair in placed.windows(2) {
2469 assert_ne!(
2470 (pair[0].0, pair[0].1),
2471 (pair[1].0, pair[1].1),
2472 "{} and {} share a position",
2473 pair[0].2,
2474 pair[1].2
2475 );
2476 }
2477 }
2478
2479 #[test]
2480 fn a_quoted_empty_key_still_opens_its_element() {
2481 let source = concat!(
2486 "---\n", "list:\n", " - \"\": K\n", " - '': L\n", " - \"\\n\": M\n", " - 2\n", "flow: [\"\": K, '': L, \"\\n\": M]\n", "---\n", "# Title\n",
2495 );
2496 let document = parse_markdown(source, MarkdownOptions::default());
2497
2498 let DocumentFrontmatter::Mapping { value, anchors, .. } = &document.frontmatter else {
2499 panic!("expected parsed frontmatter: {document:?}")
2500 };
2501 let anchor = |pointer: &str| {
2502 anchors
2503 .get(pointer)
2504 .map(|anchor| (anchor.line, anchor.column))
2505 };
2506
2507 assert_eq!(
2508 value.get("list"),
2509 Some(&serde_json::json!([{"": "K"}, {"": "L"}, {"\n": "M"}, 2])),
2510 "each element is a mapping under an empty key"
2511 );
2512 assert_eq!(anchor("/list/0"), Some((3, 5)));
2514 assert_eq!(anchor("/list/1"), Some((4, 5)));
2515 assert_eq!(anchor("/list/2"), Some((5, 5)));
2516 assert_eq!(anchor("/list/3"), Some((6, 5)));
2517 assert_eq!(anchor("/list/0/"), Some((3, 5)));
2522 assert_eq!(anchor("/list/1/"), Some((4, 5)));
2523 assert_eq!(anchor("/list/2/\n"), Some((5, 5)));
2524 assert_eq!(anchor("/flow/0"), Some((7, 8)));
2528 assert_eq!(anchor("/flow/1"), Some((7, 15)));
2529 assert_eq!(anchor("/flow/2"), Some((7, 22)));
2530 assert_eq!(
2531 source.lines().nth(6).map(|line| (
2532 line.as_bytes().get(7),
2533 line.as_bytes().get(14),
2534 line.as_bytes().get(21)
2535 )),
2536 Some((Some(&b'"'), Some(&b'\''), Some(&b'"'))),
2537 "the anchored positions hold the opening quotes"
2538 );
2539
2540 assert_distinct_anchors(source, anchors);
2541 }
2542
2543 #[test]
2544 fn line_cursor_measures_forward_without_rescanning() {
2545 let mut cursor = LineCursor::new(2, "ää: [1, 2]");
2549 assert_eq!(cursor.byte_column(1), Some(1));
2550 assert_eq!(cursor.byte_column(6), Some(8));
2551 assert_eq!(cursor.byte_column(9), Some(11));
2552 assert_eq!(cursor.byte_column(6), None);
2553 assert_eq!(cursor.byte_column(64), None);
2555
2556 let mut cursor = LineCursor::new(2, "ab");
2559 assert_eq!(cursor.byte_column(3), Some(3));
2560 assert_eq!(LineCursor::new(2, "ab").byte_column(4), None);
2561 assert_eq!(LineCursor::new(2, "ab").byte_column(0), None);
2563 }
2564
2565 #[test]
2566 fn first_column_anchors_survive_the_zero_based_parser() {
2567 let document = parse_markdown(
2574 "---\na: 1\nb: 2\n---\n# Title\n",
2575 MarkdownOptions::default(),
2576 );
2577 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2578 panic!("expected parsed frontmatter: {document:?}")
2579 };
2580 assert_eq!(
2581 anchors.get("/a"),
2582 Some(FrontmatterAnchor { line: 2, column: 1 })
2583 );
2584 assert_eq!(
2585 anchors.get("/b"),
2586 Some(FrontmatterAnchor { line: 3, column: 1 })
2587 );
2588 }
2589
2590 #[test]
2591 fn tagged_and_aliased_frontmatter_keep_their_anchors() {
2592 let document = parse_markdown(
2598 "---\ncount: !!str 5\n---\n# Title\n",
2599 MarkdownOptions::default(),
2600 );
2601 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2602 panic!("expected parsed frontmatter: {document:?}")
2603 };
2604 assert_eq!(
2605 anchors.get("/count"),
2606 Some(FrontmatterAnchor { line: 2, column: 1 })
2607 );
2608
2609 let document = parse_markdown(
2610 "---\nanchored: &a 1\nalias: *a\n---\n# Title\n",
2611 MarkdownOptions::default(),
2612 );
2613 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2614 panic!("expected parsed frontmatter: {document:?}")
2615 };
2616 assert_eq!(
2617 anchors.get("/anchored"),
2618 Some(FrontmatterAnchor { line: 2, column: 1 })
2619 );
2620 assert_eq!(
2621 anchors.get("/alias"),
2622 Some(FrontmatterAnchor { line: 3, column: 1 })
2623 );
2624 }
2625
2626 #[test]
2627 fn alias_expansions_anchor_at_the_alias_site() {
2628 let source = concat!(
2635 "---\n", "base: &x\n", " bad: \"oops\"\n", " tags:\n", " - 1\n", "ref: *x\n", "---\n",
2642 "# Title\n",
2643 );
2644 let document = parse_markdown(source, MarkdownOptions::default());
2645 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2646 panic!("expected parsed frontmatter: {document:?}")
2647 };
2648 let anchor = |pointer: &str| {
2649 anchors
2650 .get(pointer)
2651 .map(|anchor| (anchor.line, anchor.column))
2652 };
2653
2654 assert_eq!(anchor("/base"), Some((2, 1)));
2656 assert_eq!(anchor("/base/bad"), Some((3, 3)));
2657 assert_eq!(anchor("/base/tags"), Some((4, 3)));
2658 assert_eq!(anchor("/base/tags/0"), Some((5, 7)));
2659 assert_eq!(anchor("/ref"), Some((6, 1)));
2662 assert_eq!(anchor("/ref/bad"), Some((6, 6)));
2663 assert_eq!(anchor("/ref/tags"), Some((6, 6)));
2664 assert_eq!(anchor("/ref/tags/0"), Some((6, 6)));
2665 }
2666
2667 #[test]
2668 fn chained_alias_expansions_anchor_at_the_outermost_alias_site() {
2669 let source = concat!(
2678 "---\n", "leaf: &l\n", " bad: nope\n", "mid: &m\n", " inner: *l\n", "outer: *m\n", "---\n",
2685 "# Title\n",
2686 );
2687 let document = parse_markdown(source, MarkdownOptions::default());
2688 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2689 panic!("expected parsed frontmatter: {document:?}")
2690 };
2691 let anchor = |pointer: &str| {
2692 anchors
2693 .get(pointer)
2694 .map(|anchor| (anchor.line, anchor.column))
2695 };
2696 assert_eq!(anchor("/outer"), Some((6, 1)));
2697 assert_eq!(anchor("/outer/inner"), Some((6, 8)));
2698 assert_eq!(anchor("/outer/inner/bad"), Some((6, 8)));
2700
2701 let source = concat!(
2704 "---\n", "a: &p [bad]\n", "b: &q [*p]\n", "c: *q\n", "---\n",
2709 "# Title\n",
2710 );
2711 let document = parse_markdown(source, MarkdownOptions::default());
2712 let DocumentFrontmatter::Mapping { anchors, .. } = &document.frontmatter else {
2713 panic!("expected parsed frontmatter: {document:?}")
2714 };
2715 let anchor = |pointer: &str| {
2716 anchors
2717 .get(pointer)
2718 .map(|anchor| (anchor.line, anchor.column))
2719 };
2720 assert_eq!(anchor("/c"), Some((4, 1)));
2721 assert_eq!(anchor("/c/0"), Some((4, 4)));
2722 assert_eq!(anchor("/c/0/0"), Some((4, 4)));
2725 }
2726
2727 #[test]
2728 fn positions_invalid_or_unclosed_frontmatter() {
2729 let scalar = parse_markdown("---\nvalue\n---\n# Title\n", MarkdownOptions::default());
2730 let DocumentFrontmatter::Invalid { location, .. } = scalar.frontmatter else {
2731 panic!("scalar frontmatter must be invalid")
2732 };
2733 assert_eq!((location.start_line, location.end_line), (1, 3));
2734
2735 let unclosed = parse_markdown("---\nkey: value\n", MarkdownOptions::default());
2736 let DocumentFrontmatter::Invalid { location, .. } = unclosed.frontmatter else {
2737 panic!("unclosed frontmatter must be invalid")
2738 };
2739 assert_eq!((location.start_line, location.end_line), (1, 3));
2740 assert!(unclosed.sections.is_empty());
2741 }
2742
2743 #[test]
2744 fn empty_and_comment_only_frontmatter_are_not_mappings() {
2745 for source in [
2749 "---\n---\n",
2750 "---\n\n---\n",
2751 "---\n \n---\n",
2752 "---\n\t\n---\n",
2753 "---\n# comment only\n---\n",
2754 "---\n\n# comment after a blank line\n\n---\n",
2755 ] {
2756 let document = parse_markdown(source, MarkdownOptions::default());
2757 let DocumentFrontmatter::Invalid { location, message } = document.frontmatter else {
2758 panic!("empty YAML content must not become a mapping: {document:?}")
2759 };
2760 assert_eq!(message, "frontmatter must be a YAML mapping");
2761 assert_eq!(location.start_line, 1);
2762 assert_eq!(location.end_line, source.lines().count() as u64);
2763 }
2764
2765 for source in ["---\n{}\n---\n", "---\n{ }\n---\n"] {
2766 let explicit_mapping = parse_markdown(source, MarkdownOptions::default());
2767 let DocumentFrontmatter::Mapping { value, .. } = explicit_mapping.frontmatter else {
2768 panic!("an explicit empty mapping remains valid: {explicit_mapping:?}")
2769 };
2770 assert_eq!(value, serde_json::Map::new());
2771 }
2772 }
2773
2774 #[test]
2775 fn frontmatter_holding_a_second_document_is_invalid() {
2776 for source in [
2780 "---\na: 1\n...\nb: 2\n---\n",
2781 "---\na: 1\n...\nplain scalar\n---\n",
2782 ] {
2783 let document = parse_markdown(source, MarkdownOptions::default());
2784 let DocumentFrontmatter::Invalid { message, .. } = document.frontmatter else {
2785 panic!("a second frontmatter document must be invalid: {document:?}")
2786 };
2787 assert_eq!(
2788 message,
2789 "frontmatter must be a single YAML document: \
2790 a second one opens at byte 9 line 3 column 1"
2791 );
2792 }
2793
2794 let document = parse_markdown(
2797 "---\na: 1\n...\n%YAML 1.2\n---\n",
2798 MarkdownOptions::default(),
2799 );
2800 let DocumentFrontmatter::Invalid { message, .. } = document.frontmatter else {
2801 panic!("unreadable content after the document must be invalid: {document:?}")
2802 };
2803 assert_eq!(message, "frontmatter must be a single YAML document");
2804
2805 let single = parse_markdown("---\na: 1\n...\n---\n", MarkdownOptions::default());
2807 let DocumentFrontmatter::Mapping { value, .. } = single.frontmatter else {
2808 panic!("a terminated single document remains valid: {single:?}")
2809 };
2810 assert_eq!(value["a"], serde_json::json!(1));
2811 }
2812
2813 #[test]
2814 fn a_merge_key_is_an_ordinary_frontmatter_entry() {
2815 let aliased = parse_markdown(
2823 "---\nbase: &b\n a: 1\nmerged:\n <<: *b\n b: 2\n---\n",
2824 MarkdownOptions::default(),
2825 );
2826 let DocumentFrontmatter::Mapping { value, .. } = aliased.frontmatter else {
2827 panic!("a merge key parses as an ordinary mapping: {aliased:?}")
2828 };
2829 assert_eq!(
2830 value["merged"],
2831 serde_json::json!({ "<<": { "a": 1 }, "b": 2 }),
2832 );
2833
2834 let inline = parse_markdown("---\n<<: {a: 1}\nb: 2\n---\n", MarkdownOptions::default());
2837 let DocumentFrontmatter::Mapping { value, anchors, .. } = inline.frontmatter else {
2838 panic!("a merge key parses as an ordinary mapping: {inline:?}")
2839 };
2840 assert_eq!(
2841 serde_json::Value::Object(value),
2842 serde_json::json!({ "<<": { "a": 1 }, "b": 2 }),
2843 );
2844 assert_eq!(
2845 anchors.get("/<<"),
2846 Some(FrontmatterAnchor { line: 2, column: 1 }),
2847 );
2848 }
2849
2850 #[test]
2851 fn recursive_frontmatter_aliases_terminate() {
2852 for source in [
2856 "---\na: &x [*x]\n---\n",
2857 "---\na: &x {k: *x}\n---\n",
2858 "---\na: &x [[[*x]]]\n---\n",
2859 "---\na: &x [*y]\nb: &y [*x]\n---\n",
2860 ] {
2861 let document = parse_markdown(source, MarkdownOptions::default());
2862 assert!(
2863 matches!(document.frontmatter, DocumentFrontmatter::Invalid { .. }),
2864 "recursive alias was accepted: {source:?}"
2865 );
2866 }
2867
2868 let document = parse_markdown("---\na: &x [1]\nb: *x\n---\n", MarkdownOptions::default());
2870 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
2871 panic!("a backward alias remains valid: {document:?}")
2872 };
2873 assert_eq!(value["b"], serde_json::json!([1]));
2874 }
2875
2876 fn alias_bomb_frontmatter(depth: usize) -> String {
2884 let mut bomb = String::from("---\na0: &x0 [1,1,1,1]\n");
2885 for level in 1..=depth {
2886 let alias = format!("*x{}", level - 1);
2887 bomb.push_str(&format!(
2888 "a{level}: &x{level} [{alias},{alias},{alias},{alias}]\n"
2889 ));
2890 }
2891 bomb.push_str("---\n# Title\n");
2892 bomb
2893 }
2894
2895 #[test]
2896 fn frontmatter_alias_expansion_is_bounded() {
2897 for depth in [9, 12, 15] {
2905 let bomb = alias_bomb_frontmatter(depth);
2906 let started = std::time::Instant::now();
2907 let document = parse_markdown(&bomb, MarkdownOptions::default());
2908 let elapsed = started.elapsed();
2909 let DocumentFrontmatter::Invalid { location, message } = document.frontmatter else {
2913 panic!("an alias bomb at depth {depth} must be rejected")
2914 };
2915 assert_eq!(
2916 message,
2917 "frontmatter expands YAML aliases beyond its size limit"
2918 );
2919 assert_eq!(
2920 (location.start_line, location.end_line),
2921 (1, depth as u64 + 3)
2922 );
2923 assert!(
2924 elapsed < std::time::Duration::from_secs(1),
2925 "an alias bomb at depth {depth} took {elapsed:?}, so it was expanded before being refused"
2926 );
2927 }
2928
2929 let mut reused = String::from("---\nbase: &base [1, 2, 3]\n");
2932 for entry in 0..10 {
2933 reused.push_str(&format!("copy{entry}: *base\n"));
2934 }
2935 reused.push_str("---\n# Title\n");
2936 let document = parse_markdown(&reused, MarkdownOptions::default());
2937 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
2938 panic!("repeated aliases to one node remain valid: {document:?}")
2939 };
2940 assert_eq!(value["copy9"], serde_json::json!([1, 2, 3]));
2941 }
2942
2943 fn deeply_nested_frontmatter(levels: usize, tagged: bool) -> String {
2949 let tag = if tagged { "tag: !!str x\n" } else { "" };
2950 format!("---\n{tag}deep:\n {}1\n---\n# Title\n", "- ".repeat(levels))
2951 }
2952
2953 fn innermost_sequence(
2955 value: &serde_json::Map<String, serde_json::Value>,
2956 levels: usize,
2957 ) -> &serde_json::Value {
2958 let mut node = &value["deep"];
2959 for _ in 1..levels {
2960 node = &node[0];
2961 }
2962 node
2963 }
2964
2965 #[test]
2966 fn frontmatter_nesting_is_bounded() {
2967 let levels = MAX_YAML_DEPTH - 1;
2969 let document = parse_markdown(
2970 &deeply_nested_frontmatter(levels, false),
2971 MarkdownOptions::default(),
2972 );
2973 let DocumentFrontmatter::Mapping { value, anchors, .. } = document.frontmatter else {
2974 panic!("nesting within the limit stays valid: {document:?}")
2975 };
2976 assert_eq!(innermost_sequence(&value, levels)[0], serde_json::json!(1));
2977 let document = parse_markdown(
2980 &deeply_nested_frontmatter(levels, true),
2981 MarkdownOptions::default(),
2982 );
2983 let DocumentFrontmatter::Mapping {
2984 value,
2985 anchors: tagged_anchors,
2986 ..
2987 } = document.frontmatter
2988 else {
2989 panic!("nesting within the limit stays valid when tagged: {document:?}")
2990 };
2991 assert_eq!(innermost_sequence(&value, levels)[0], serde_json::json!(1));
2992 assert!(!anchors.is_empty() && !tagged_anchors.is_empty());
2993
2994 for levels in [MAX_YAML_DEPTH, 30_000] {
2997 for tagged in [false, true] {
2998 let source = deeply_nested_frontmatter(levels, tagged);
2999 let document = parse_markdown(&source, MarkdownOptions::default());
3000 let DocumentFrontmatter::Invalid { location, message } = document.frontmatter
3001 else {
3002 panic!("nesting past the limit must be rejected: {levels} levels, {tagged}")
3003 };
3004 assert_eq!(message, "frontmatter nests YAML beyond its depth limit");
3005 assert_eq!(location.start_line, 1);
3006 }
3007 }
3008 }
3009
3010 fn alias_deepened_frontmatter(lines: usize, levels: usize) -> String {
3020 let (open, close) = ("[".repeat(levels), "]".repeat(levels));
3021 let mut source = format!("---\na0: &x0 {open}1{close}\n");
3022 for line in 1..lines {
3023 source.push_str(&format!("a{line}: &x{line} {open}*x{}{close}\n", line - 1));
3024 }
3025 source.push_str("---\n# Title\n");
3026 source
3027 }
3028
3029 #[test]
3030 fn alias_expanded_nesting_is_bounded() {
3031 for (lines, levels) in [(70, 127), (2_000, 127), (MAX_YAML_DEPTH, 1)] {
3041 let source = alias_deepened_frontmatter(lines, levels);
3042 let started = std::time::Instant::now();
3043 let document = parse_markdown(&source, MarkdownOptions::default());
3044 let elapsed = started.elapsed();
3045 let DocumentFrontmatter::Invalid { message, .. } = document.frontmatter else {
3046 panic!("{lines} lines of {levels} alias-expanded levels were accepted")
3047 };
3048 assert_eq!(message, "frontmatter nests YAML beyond its depth limit");
3049 assert!(
3050 elapsed < std::time::Duration::from_secs(5),
3051 "{lines} lines of {levels} levels took {elapsed:?}, so the tree was built first"
3052 );
3053 }
3054
3055 let source = alias_deepened_frontmatter(MAX_YAML_DEPTH - 1, 1);
3060 let document = parse_markdown(&source, MarkdownOptions::default());
3061 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3062 panic!("alias-expanded nesting that fills the limit is built: {document:?}")
3063 };
3064 let mut node = &value[&format!("a{}", MAX_YAML_DEPTH - 2)];
3065 for _ in 1..MAX_YAML_DEPTH - 1 {
3066 node = &node[0];
3067 }
3068 assert_eq!(node[0], serde_json::json!(1));
3069 }
3070
3071 #[test]
3072 fn alias_spliced_depth_just_past_the_limit_is_a_refusal_not_a_crash() {
3073 let source = alias_deepened_frontmatter(MAX_YAML_DEPTH + 2, 1);
3082 assert_eq!(
3083 expect_invalid_frontmatter(&source),
3084 "frontmatter nests YAML beyond its depth limit"
3085 );
3086 }
3087
3088 fn key_deepened_frontmatter(lines: usize) -> String {
3096 let mut source = String::from("---\na0: &a0 {x: y}\n");
3097 for line in 1..lines {
3098 source.push_str(&format!("a{line}: &a{line} {{? [*a{}] : v}}\n", line - 1));
3099 }
3100 source.push_str("---\n# Title\n");
3101 source
3102 }
3103
3104 #[test]
3105 fn alias_nesting_reached_through_a_mapping_key_is_bounded() {
3106 assert_eq!(
3119 expect_invalid_frontmatter(&key_deepened_frontmatter(70)),
3120 "frontmatter nests YAML beyond its depth limit"
3121 );
3122 assert_eq!(
3123 expect_invalid_frontmatter(&key_deepened_frontmatter(4)),
3124 "frontmatter mapping keys must be strings"
3125 );
3126 }
3127
3128 #[test]
3129 fn nesting_depth_counts_collections_that_are_open_at_once() {
3130 let mut wide = String::from("---\n");
3133 for entry in 0..MAX_YAML_DEPTH * 2 {
3134 wide.push_str(&format!("key{entry}: [1, 2, 3]\n"));
3135 }
3136 wide.push_str("---\n");
3137 let document = parse_markdown(&wide, MarkdownOptions::default());
3138 assert!(matches!(
3139 document.frontmatter,
3140 DocumentFrontmatter::Mapping { .. }
3141 ));
3142 }
3143
3144 #[test]
3145 fn the_exact_builder_bounds_its_own_recursion() {
3146 let nested = |levels: usize| format!("deep:\n {}1\n", "- ".repeat(levels));
3152 let (filled, _) = exact_frontmatter_mapping(&nested(MAX_YAML_DEPTH - 1), NO_MARK)
3153 .expect("nesting that fills the limit is built");
3154 assert_eq!(innermost_sequence(&filled, MAX_YAML_DEPTH - 1)[0], 1);
3155 for levels in [MAX_YAML_DEPTH, MAX_YAML_DEPTH + 1] {
3156 assert_eq!(
3157 exact_frontmatter_mapping(&nested(levels), NO_MARK),
3158 Err("frontmatter nests YAML beyond its depth limit".to_owned()),
3159 "the builder accepted {levels} levels of its own accord"
3160 );
3161 }
3162 }
3163
3164 #[test]
3165 fn the_exact_builder_rejects_a_key_repeated_in_any_spelling() {
3166 for duplicate in [
3174 "a: 1\na: 2\n",
3175 "a: 1\n\"a\": 2\n",
3176 "\"a\": 1\na: 2\n",
3177 "'a': 1\n\"a\": 2\n",
3178 "a: 1\nb:\n c: 1\n c: 2\n",
3179 "a: {b: 1, b: 2}\n",
3180 "a:\n - {k: 1, k: 2}\n",
3181 "a: !!str x\nb: 1\nb: 2\n",
3182 "? &k a\n: 1\n? *k\n: 2\n",
3183 "? [x]\n: 1\n? [x]\n: 2\n",
3184 ] {
3185 assert_eq!(
3186 exact_frontmatter_mapping(duplicate, NO_MARK),
3187 Err("frontmatter contains a duplicate mapping key".to_owned()),
3188 "a duplicate key was accepted: {duplicate:?}"
3189 );
3190 }
3191
3192 for valid in [
3196 "a:\n - {k: 1}\n - {k: 2}\n",
3197 "a: {k: 1}\nb: {k: 2}\n",
3198 "a:\n k: 1\nb:\n k: 2\n",
3199 ] {
3200 assert!(
3201 exact_frontmatter_mapping(valid, NO_MARK).is_ok(),
3202 "distinct mappings sharing a key name were rejected: {valid:?}"
3203 );
3204 }
3205
3206 assert_eq!(
3212 exact_frontmatter_mapping("a: 1\n? [x]\n: 2\n", NO_MARK),
3213 Err("frontmatter mapping keys must be strings".to_owned())
3214 );
3215 assert_eq!(
3216 exact_frontmatter_mapping("a: !!int 1.0\na: 2\n", NO_MARK),
3217 Err("frontmatter contains a duplicate mapping key".to_owned())
3218 );
3219 assert_eq!(
3220 exact_frontmatter_mapping("a: !!int 1.0\n\"a\": 2\n", NO_MARK),
3221 Err("frontmatter contains an invalid explicitly tagged integer".to_owned())
3222 );
3223 }
3224
3225 #[test]
3226 fn the_exact_builder_reads_tags_on_collections_as_well_as_scalars() {
3227 for (source, expected) in [
3232 ("a: !!seq [one, two]\n", serde_json::json!(["one", "two"])),
3233 ("a: !!seq\n - one\n", serde_json::json!(["one"])),
3234 ("a: !!map {one: two}\n", serde_json::json!({"one": "two"})),
3235 ("a: !!map\n one: two\n", serde_json::json!({"one": "two"})),
3236 ("a: !custom [one]\n", serde_json::json!(["one"])),
3239 ("a: !custom {one: two}\n", serde_json::json!({"one": "two"})),
3240 ] {
3241 let (mapping, _) = exact_frontmatter_mapping(source, NO_MARK)
3242 .unwrap_or_else(|error| panic!("{source:?}: {error}"));
3243 assert_eq!(mapping["a"], expected, "{source:?}");
3244 }
3245
3246 for (source, expected) in [
3247 ("a: !!map [one, two]\n", "seq"),
3248 ("a: !!str [one]\n", "seq"),
3249 ("a: !!seq {one: two}\n", "map"),
3250 ("a: !!str {one: two}\n", "map"),
3251 ("!!str\na: 1\n", "map"),
3253 ] {
3254 assert_eq!(
3255 exact_frontmatter_mapping(source, NO_MARK),
3256 Err(format!(
3257 "frontmatter contains an invalid tag for a YAML {expected}"
3258 )),
3259 "{source:?}"
3260 );
3261 }
3262
3263 for (source, expected) in [
3266 ("a: !!str 123\n", serde_json::json!("123")),
3267 ("a: !!int \"42\"\n", serde_json::json!(42)),
3268 ("a: !!bool TRUE\n", serde_json::json!(true)),
3269 ("a: !!null ~\n", serde_json::Value::Null),
3270 ("a: !!unknown 1\n", serde_json::json!("1")),
3271 ("a: !thing 123\n", serde_json::json!(123)),
3274 ] {
3275 let (mapping, _) = exact_frontmatter_mapping(source, NO_MARK)
3276 .unwrap_or_else(|error| panic!("{source:?}: {error}"));
3277 assert_eq!(mapping["a"], expected, "{source:?}");
3278 }
3279 assert_eq!(
3280 exact_frontmatter_mapping("a: !!str [one, two]\n", NO_MARK),
3281 Err("frontmatter contains an invalid tag for a YAML seq".to_owned())
3282 );
3283 }
3284
3285 #[test]
3286 fn the_exact_builder_keeps_a_quoted_scalar_a_string() {
3287 let entries = "a: \"1\"\nb: 'true'\nc: \"null\"\nd: |\n 1\ne: 1\n";
3299 let tagged = expect_frontmatter_mapping(&format!("---\n{entries}f: !!str y\n---\n"));
3300 assert_eq!(tagged["a"], serde_json::json!("1"));
3301 assert_eq!(tagged["b"], serde_json::json!("true"));
3302 assert_eq!(tagged["c"], serde_json::json!("null"));
3303 assert_eq!(tagged["d"], serde_json::json!("1\n"));
3304 assert_eq!(tagged["e"], serde_json::json!(1));
3305
3306 let untagged = expect_frontmatter_mapping(&format!("---\n{entries}---\n"));
3312 for key in ["a", "b", "c", "d", "e"] {
3313 assert_eq!(
3314 tagged[key], untagged[key],
3315 "a tag changed the resolution of {key}"
3316 );
3317 }
3318 }
3319
3320 #[test]
3321 fn the_exact_builder_refuses_a_second_document_itself() {
3322 for (source, position) in [
3342 ("a: 1\n--- \nb: 2\n", "byte 5 line 2 column 1"),
3343 ("a: &x 1\n--- \nb: *x\n", "byte 8 line 2 column 1"),
3344 ("a: 1\n...\nb: 2\n", "byte 9 line 3 column 1"),
3345 ("a: &x 1\n...\nb: *missing\n", "byte 12 line 3 column 1"),
3346 ("a: &x 1\n...\nb: *x\n", "byte 12 line 3 column 1"),
3347 ] {
3348 assert_eq!(
3349 exact_frontmatter_mapping(source, NO_MARK),
3350 Err(format!(
3351 "frontmatter must be a single YAML document: a second one opens at {position}"
3352 )),
3353 "a second document was read: {source:?}"
3354 );
3355 }
3356 }
3357
3358 #[test]
3359 fn the_alias_budget_allows_a_hundred_nodes_per_event() {
3360 let sequence = vec!["1"; 1000].join(",");
3373 let block = |lines: usize| {
3374 let mut source = format!("---\nbase: &b [{sequence}]\n");
3375 for line in 0..lines {
3376 source.push_str(&format!("copy{line}: *b\n"));
3377 }
3378 source.push_str("---\n# Title\n");
3379 source
3380 };
3381 let built = expect_frontmatter_mapping(&block(124));
3382 assert_eq!(built["copy123"][999], serde_json::json!(1));
3383 assert_eq!(
3384 expect_invalid_frontmatter(&block(125)),
3385 "frontmatter expands YAML aliases beyond its size limit"
3386 );
3387 }
3388
3389 #[test]
3390 fn duplicate_key_detection_does_not_compare_every_pair_of_keys() {
3391 const KEYS: usize = 2_000;
3410 let mut source = format!("---\nbig: &b [{}]\n", vec!["1"; 450].join(","));
3411 for key in 0..KEYS {
3412 source.push_str(&format!("? [*b,{key}]\n: {key}\n"));
3413 }
3414 source.push_str("---\n# Title\n");
3415 KEY_COMPARISONS.with(|made| made.set(0));
3416 let started = std::time::Instant::now();
3417 let message = expect_invalid_frontmatter(&source);
3418 let elapsed = started.elapsed();
3419 let compared = KEY_COMPARISONS.with(std::cell::Cell::get);
3420 assert_eq!(message, "frontmatter mapping keys must be strings");
3424 assert!(
3425 compared < KEYS,
3426 "{KEYS} distinct collection keys cost {compared} whole-node comparisons"
3427 );
3428 assert!(
3429 elapsed < std::time::Duration::from_secs(5),
3430 "two thousand collection keys took {elapsed:?} to compare"
3431 );
3432 }
3433
3434 #[test]
3435 fn frontmatter_syntax_errors_carry_the_parser_position() {
3436 for (body, message) in [
3453 ("title: Doc\n]\n", "misplaced bracket at byte 11 line 2 column 1"),
3454 ("*x]\n", "misplaced bracket at byte 2 line 1 column 3"),
3455 (
3456 "a: [1, 2\n",
3457 "while parsing a flow sequence, expected ',' or ']' at byte 9 line 2 column 1",
3458 ),
3459 (
3460 "{a: 1\n",
3461 "while parsing a flow mapping, did not find expected ',' or '}' \
3462 at byte 6 line 2 column 1",
3463 ),
3464 (
3465 "tags: [, draft]\n",
3466 "while parsing a node, did not find expected node content at byte 7 line 1 column 8",
3467 ),
3468 (
3469 "a: *nope\n",
3470 "while parsing node, found unknown anchor at byte 3 line 1 column 4",
3471 ),
3472 (
3473 "title: 'unterminated\n",
3474 "while scanning a quoted scalar, found unexpected end of stream \
3475 at byte 7 line 1 column 8",
3476 ),
3477 (
3478 "title: \"\\q\"\n",
3479 "while parsing a quoted scalar, found unknown escape character \
3480 at byte 7 line 1 column 8",
3481 ),
3482 (
3483 "a:\n b: 1\n c: 2\n",
3484 "while parsing a block mapping, did not find expected key at byte 11 line 3 column 2",
3485 ),
3486 (
3487 "a: 1\n b: 2\n",
3488 "mapping values are not allowed in this context at byte 7 line 2 column 3",
3489 ),
3490 ("a: 1\nb\n", "simple key expect ':' at byte 7 line 3 column 1"),
3491 (
3492 "é: 'x\n",
3493 "while scanning a quoted scalar, found unexpected end of stream \
3494 at byte 3 line 1 column 4",
3495 ),
3496 ] {
3497 assert_eq!(
3498 expect_invalid_frontmatter(&format!("---\n{body}---\n# Title\n")),
3499 format!("invalid YAML frontmatter: {message}"),
3500 "{body:?}"
3501 );
3502 }
3503 }
3504
3505 fn expect_frontmatter_mapping(source: &str) -> serde_json::Map<String, serde_json::Value> {
3508 let document = parse_markdown(source, MarkdownOptions::default());
3509 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3510 panic!("frontmatter must parse as a mapping: {source:?}")
3511 };
3512 value
3513 }
3514
3515 fn expect_invalid_frontmatter(source: &str) -> String {
3517 let document = parse_markdown(source, MarkdownOptions::default());
3518 let DocumentFrontmatter::Invalid { message, .. } = document.frontmatter else {
3519 panic!("frontmatter must be refused: {source:?}")
3520 };
3521 message
3522 }
3523
3524 #[test]
3525 fn frontmatter_drops_one_leading_byte_order_mark() {
3526 for tag in ["", "!!int "] {
3539 let marked = format!("---\n\u{feff}version: {tag}1\nx: 2\n---\n");
3540 let plain = format!("---\nversion: {tag}1\nx: 2\n---\n");
3541 let document = parse_markdown(&marked, MarkdownOptions::default());
3542 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3543 panic!("a leading mark is dropped: {marked:?}")
3544 };
3545 assert_eq!(value, expect_frontmatter_mapping(&plain), "{marked:?}");
3546
3547 let doubled = format!("---\n\u{feff}\u{feff}version: {tag}1\n---\n");
3550 let doubled = expect_frontmatter_mapping(&doubled);
3551 assert_eq!(doubled.keys().collect::<Vec<_>>(), ["\u{feff}version"]);
3552
3553 let inside = format!("---\nx: {tag}2\na: \u{feff}1\n---\n");
3559 assert_eq!(expect_frontmatter_mapping(&inside)["a"], "\u{feff}1");
3560 }
3561
3562 let document = parse_markdown(
3568 "---\n\u{feff}version: 1\nx: 2\n---\n",
3569 MarkdownOptions::default(),
3570 );
3571 let DocumentFrontmatter::Mapping { anchors, .. } = document.frontmatter else {
3572 panic!("a marked block still parses")
3573 };
3574 assert_eq!(
3575 anchors.get("/version"),
3576 Some(FrontmatterAnchor { line: 2, column: 4 }),
3577 );
3578 assert_eq!(
3580 anchors.get("/x"),
3581 Some(FrontmatterAnchor { line: 3, column: 1 }),
3582 );
3583
3584 let empty = parse_markdown("---\n\u{feff}\n---\n", MarkdownOptions::default());
3590 let DocumentFrontmatter::Invalid { message, .. } = empty.frontmatter else {
3591 panic!("a block holding only a mark holds no mapping: {empty:?}")
3592 };
3593 assert_eq!(message, "frontmatter must be a YAML mapping");
3594 for tag in ["", "!!str "] {
3595 let marked = format!("---\n\u{feff}...\nb: {tag}2\n---\n");
3596 let plain = format!("---\n...\nb: {tag}2\n---\n");
3597 assert_eq!(
3598 expect_frontmatter_mapping(&marked),
3599 expect_frontmatter_mapping(&plain),
3600 "a mark changed how a document boundary was read"
3601 );
3602 }
3603
3604 let marked = expect_invalid_frontmatter("---\n\u{feff}title: 'unterminated\n---\n");
3609 let plain = expect_invalid_frontmatter("---\ntitle: 'unterminated\n---\n");
3610 assert_eq!(
3611 plain,
3612 "invalid YAML frontmatter: while scanning a quoted scalar, \
3613 found unexpected end of stream at byte 7 line 1 column 8"
3614 );
3615 assert_eq!(
3616 marked,
3617 "invalid YAML frontmatter: while scanning a quoted scalar, \
3618 found unexpected end of stream at byte 8 line 1 column 9"
3619 );
3620 assert_eq!(
3623 expect_invalid_frontmatter("---\n\u{feff}a: 1\nb: 'x\n---\n"),
3624 "invalid YAML frontmatter: while scanning a quoted scalar, \
3625 found unexpected end of stream at byte 9 line 2 column 4"
3626 );
3627 assert_eq!(
3628 expect_invalid_frontmatter("---\na: 1\nb: 'x\n---\n"),
3629 "invalid YAML frontmatter: while scanning a quoted scalar, \
3630 found unexpected end of stream at byte 8 line 2 column 4"
3631 );
3632 }
3633
3634 #[test]
3635 fn rejects_non_string_frontmatter_mapping_keys() {
3636 let document = parse_markdown("---\n1: value\n---\n", MarkdownOptions::default());
3637 let DocumentFrontmatter::Invalid { message, .. } = document.frontmatter else {
3638 panic!("numeric mapping key must be invalid")
3639 };
3640 assert!(message.contains("keys must be strings"));
3641 }
3642
3643 #[test]
3644 fn preserves_arbitrary_precision_frontmatter_numbers() {
3645 let document = parse_markdown(
3646 "---\nbig: 184467440737095516160\nprecise: 0.123456789012345678901234567890\nquoted: \"184467440737095516160\"\n---\n",
3647 MarkdownOptions::default(),
3648 );
3649 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3650 panic!("expected valid numeric frontmatter: {document:?}")
3651 };
3652 assert_eq!(value["big"].to_string(), "184467440737095516160");
3653 assert_eq!(
3654 value["precise"].to_string(),
3655 "0.123456789012345678901234567890"
3656 );
3657 assert_eq!(value["quoted"], "184467440737095516160");
3658 }
3659
3660 #[test]
3661 fn preserves_json_compatible_frontmatter_number_spellings_and_typed_identity() {
3662 let document = parse_markdown(
3663 concat!(
3664 "---\n",
3665 "whole: 100.0\n",
3666 "integer: 100\n",
3667 "fraction: 1.5\n",
3668 "lower_exponent: 1e2\n",
3669 "upper_exponent: 1E2\n",
3670 "tagged: !!float 2.50\n",
3671 "base: &number 3.75\n",
3672 "alias: *number\n",
3673 "normalized: +4.50\n",
3674 "forced_float: !!float 1\n",
3675 "huge: 1e10000\n",
3676 "tiny: 1e-10000\n",
3677 "unrelated: !!str value\n",
3678 "---\n",
3679 ),
3680 MarkdownOptions::default(),
3681 );
3682 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3683 panic!("expected valid numeric frontmatter: {document:?}")
3684 };
3685
3686 assert_eq!(value["whole"].to_string(), "100.0");
3687 assert_ne!(value["whole"], value["integer"]);
3688 assert!(jsonschema::draft202012::is_valid(
3689 &serde_json::json!({"const": 100}),
3690 &value["whole"]
3691 ));
3692 assert_eq!(value["fraction"].to_string(), "1.5");
3693 assert_eq!(value["lower_exponent"].to_string(), "1e2");
3694 assert_eq!(value["upper_exponent"].to_string(), "1E2");
3695 assert_eq!(value["tagged"].to_string(), "2.50");
3696 assert_eq!(value["base"].to_string(), "3.75");
3697 assert_eq!(value["alias"].to_string(), "3.75");
3698 assert_eq!(value["normalized"].to_string(), "45e-1");
3699 assert_eq!(value["forced_float"].to_string(), "1e+0");
3700 assert_ne!(value["forced_float"], serde_json::json!(1));
3701 assert_eq!(value["huge"].to_string(), "1e10000");
3702 assert_eq!(value["tiny"].to_string(), "1e-10000");
3703 }
3704
3705 #[test]
3706 fn explicit_tags_resolve_to_their_declared_types() {
3707 let document = parse_markdown(
3708 concat!(
3709 "---\n",
3710 "string: !!str 123\n",
3711 "integer: !!int \"42\"\n",
3712 "boolean: !!bool TRUE\n",
3713 "custom: !thing 123\n",
3714 "---\n",
3715 ),
3716 MarkdownOptions::default(),
3717 );
3718 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3719 panic!("expected tagged frontmatter")
3720 };
3721 assert_eq!(value["string"], "123");
3722 assert_eq!(value["integer"], 42);
3723 assert_eq!(value["boolean"], true);
3724 assert_eq!(value["custom"], 123);
3725 }
3726
3727 #[test]
3728 fn explicit_tag_on_a_sibling_does_not_round_a_decimal() {
3729 let plain = parse_markdown(
3730 "---\nprecise: 0.1234567890123456789012345\n---\n",
3731 MarkdownOptions::default(),
3732 );
3733 let tagged = parse_markdown(
3734 "---\nprecise: 0.1234567890123456789012345\ntagged: !!str abc\n---\n",
3735 MarkdownOptions::default(),
3736 );
3737 let DocumentFrontmatter::Mapping {
3738 value: plain_value, ..
3739 } = plain.frontmatter
3740 else {
3741 panic!("expected untagged frontmatter")
3742 };
3743 let DocumentFrontmatter::Mapping {
3744 value: tagged_value,
3745 ..
3746 } = tagged.frontmatter
3747 else {
3748 panic!("expected tagged frontmatter")
3749 };
3750
3751 assert_eq!(tagged_value["precise"], plain_value["precise"]);
3752 assert_eq!(tagged_value["tagged"], "abc");
3753 }
3754
3755 #[test]
3756 fn explicit_tags_preserve_oversized_integers_and_forced_number_types() {
3757 let document = parse_markdown(
3758 concat!(
3759 "---\n",
3760 "big: 184467440737095516160\n",
3761 "precise: !!float 0.1234567890123456789012345\n",
3762 "tagged: !!str 123\n",
3763 "---\n",
3764 ),
3765 MarkdownOptions::default(),
3766 );
3767 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3768 panic!("expected tagged numeric frontmatter: {document:?}")
3769 };
3770
3771 assert_eq!(value["big"].to_string(), "184467440737095516160");
3772 assert_eq!(value["precise"].to_string(), "0.1234567890123456789012345");
3773 assert_eq!(value["tagged"], "123");
3774 }
3775
3776 #[test]
3777 fn the_exact_builder_keeps_every_digit_it_was_given() {
3778 for (first, second) in [
3786 ("1234567890123456789012345", "1234567890123456789012346"),
3787 (
3788 "123456789012345678901234567890",
3789 "123456789012345678901234567891",
3790 ),
3791 ("0.1234567890123456789012345", "0.1234567890123456789012346"),
3792 (
3793 "1.23456789012345678901234567890e5",
3794 "1.23456789012345678901234567891e5",
3795 ),
3796 ] {
3797 let source = format!("first: {first}\nsecond: {second}\ntagged: !!str x\n");
3800 let (mapping, _) = exact_frontmatter_mapping(&source, NO_MARK)
3801 .unwrap_or_else(|error| panic!("{source:?}: {error}"));
3802 assert_eq!(mapping["first"].to_string(), first);
3803 assert_eq!(mapping["second"].to_string(), second);
3804 assert_ne!(mapping["first"], mapping["second"], "{source:?}");
3805 }
3806 }
3807
3808 #[test]
3809 fn standard_tags_with_mismatched_values_are_rejected() {
3810 for invalid in [
3811 "bad: !!int 1.0",
3812 "bad: !!int 01",
3813 "bad: !!float 0x2A",
3814 "bad: !!null nope",
3815 "bad: !!str [one, two]",
3816 "bad: !!seq {one: two}",
3817 "bad: !!map [one, two]",
3818 ] {
3819 let source = format!("---\nhuge: 184467440737095516160\n{invalid}\n---\n");
3820 let document = parse_markdown(&source, MarkdownOptions::default());
3821 assert!(
3822 matches!(document.frontmatter, DocumentFrontmatter::Invalid { .. }),
3823 "invalid tag was accepted: {invalid}"
3824 );
3825 }
3826 }
3827
3828 #[test]
3829 fn standard_tags_with_conforming_values_are_accepted() {
3830 let document = parse_markdown(
3831 concat!(
3832 "---\n",
3833 "huge: 184467440737095516160\n",
3834 "string: !!str 123\n",
3835 "null_value: !!null null\n",
3836 "integer: !!int 42\n",
3837 "binary: !!int 0b101010\n",
3838 "float: !!float 1.25\n",
3839 "integer_float: !!float 1\n",
3840 "sequence: !!seq [one, two]\n",
3841 "mapping: !!map {one: two}\n",
3842 "---\n",
3843 ),
3844 MarkdownOptions::default(),
3845 );
3846 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3847 panic!("expected valid explicitly tagged frontmatter: {document:?}")
3848 };
3849
3850 assert_eq!(value["huge"].to_string(), "184467440737095516160");
3851 assert_eq!(value["string"], "123");
3852 assert_eq!(value["null_value"], serde_json::Value::Null);
3853 assert_eq!(value["integer"], 42);
3854 assert_eq!(value["binary"], 42);
3855 assert_eq!(value["float"].to_string(), "1.25");
3856 assert_eq!(value["integer_float"].to_string(), "1e+0");
3857 assert_ne!(value["integer_float"], serde_json::json!(1));
3858 assert!(jsonschema::draft202012::is_valid(
3859 &serde_json::json!({"const": 1}),
3860 &value["integer_float"]
3861 ));
3862 assert_eq!(value["sequence"], serde_json::json!(["one", "two"]));
3863 assert_eq!(value["mapping"], serde_json::json!({"one": "two"}));
3864 }
3865
3866 #[test]
3867 fn huge_and_tiny_exponents_keep_their_spelling() {
3868 let document = parse_markdown(
3869 concat!(
3870 "---\n",
3871 "huge: 1e10000\n",
3872 "tiny: 1e-10000\n",
3873 "tagged_huge: !!float 2e10000\n",
3874 "tagged_tiny: !!float 2e-10000\n",
3875 "unrelated: !!str value\n",
3876 "---\n",
3877 ),
3878 MarkdownOptions::default(),
3879 );
3880 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3881 panic!("expected exact ranged decimals: {document:?}")
3882 };
3883
3884 assert_eq!(value["huge"].to_string(), "1e10000");
3885 assert_eq!(value["tiny"].to_string(), "1e-10000");
3886 assert_eq!(value["tagged_huge"].to_string(), "2e10000");
3887 assert_eq!(value["tagged_tiny"].to_string(), "2e-10000");
3888 }
3889
3890 #[test]
3891 fn nonfinite_and_malformed_float_tags_are_rejected() {
3892 for invalid in ["bad: !!float .inf", "bad: !!float 1e", "bad: !!float nope"] {
3893 let source = format!("---\nhuge: 184467440737095516160\n{invalid}\n---\n");
3894 let document = parse_markdown(&source, MarkdownOptions::default());
3895 assert!(
3896 matches!(document.frontmatter, DocumentFrontmatter::Invalid { .. }),
3897 "invalid float was accepted: {invalid}"
3898 );
3899 }
3900 }
3901
3902 #[test]
3903 fn preserves_yaml_alias_values() {
3904 let document = parse_markdown(
3905 "---\nbase: &base 42\ncopy: *base\n---\n",
3906 MarkdownOptions::default(),
3907 );
3908 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3909 panic!("expected aliased frontmatter: {document:?}")
3910 };
3911 assert_eq!(value["base"], 42);
3912 assert_eq!(value["copy"], value["base"]);
3913 }
3914
3915 #[test]
3916 fn aliases_preserve_exact_numeric_values() {
3917 let document = parse_markdown(
3918 "---\nbase: &base 0.1234567890123456789012345\ncopy: *base\n---\n",
3919 MarkdownOptions::default(),
3920 );
3921 let DocumentFrontmatter::Mapping { value, .. } = document.frontmatter else {
3922 panic!("expected aliased frontmatter: {document:?}")
3923 };
3924
3925 assert_eq!(value["base"].to_string(), "0.1234567890123456789012345");
3926 assert_eq!(value["copy"], value["base"]);
3927 }
3928
3929 #[test]
3930 fn duplicate_keys_remain_invalid_beside_a_tag() {
3931 let document = parse_markdown(
3932 "---\ntagged: !!str value\nduplicate: one\nduplicate: two\n---\n",
3933 MarkdownOptions::default(),
3934 );
3935
3936 assert!(matches!(
3937 document.frontmatter,
3938 DocumentFrontmatter::Invalid { .. }
3939 ));
3940 }
3941
3942 fn assert_valid_range(source: &str, range: TextRange) {
3943 assert!(range.start <= range.end);
3944 assert!(range.end.0 <= source.len());
3945 assert!(source.is_char_boundary(range.start.0));
3946 assert!(source.is_char_boundary(range.end.0));
3947 }
3948
3949 fn assert_valid_section_ranges(source: &str, sections: &[Section]) {
3950 for section in sections {
3951 assert_valid_range(source, section.heading.location.range);
3952 assert_valid_range(source, section.heading.location.line_range);
3953 assert!(section.heading.location.line >= 1);
3954 assert!(section.heading.location.column >= 1);
3955 assert_valid_section_ranges(source, §ion.children);
3956 }
3957 }
3958
3959 fn assert_valid_anchors(
3960 source: &str,
3961 location: &FrontmatterLocation,
3962 anchors: &FrontmatterAnchors,
3963 ) {
3964 let lines = LineIndex::new(source);
3965 for (pointer, anchor) in &anchors.0 {
3966 assert!(
3967 (2..location.end_line).contains(&anchor.line),
3968 "{pointer} left the block: {anchor:?}"
3969 );
3970 let text = lines
3971 .line_text(source, anchor.line as usize)
3972 .unwrap_or_else(|| panic!("{pointer} names a line the document lacks"));
3973 let column = anchor.column as usize - 1;
3974 assert!(
3975 column <= text.len(),
3976 "{pointer} overruns its line: {anchor:?}"
3977 );
3978 assert!(
3979 text.is_char_boundary(column),
3980 "{pointer} splits a character: {anchor:?}"
3981 );
3982 }
3983 assert_distinct_anchors(source, anchors);
3984 }
3985
3986 fn assert_distinct_anchors(source: &str, anchors: &FrontmatterAnchors) {
3994 let mut placed: Vec<_> = anchors
3995 .0
3996 .iter()
3997 .map(|(pointer, anchor)| (anchor.line, anchor.column, pointer.as_str()))
3998 .collect();
3999 placed.sort_unstable();
4000 for pair in placed.windows(2) {
4001 let (line, column, earlier) = pair[0];
4002 let (other_line, other_column, later) = pair[1];
4003 if (line, column) != (other_line, other_column) {
4004 continue;
4005 }
4006 assert!(
4007 is_pointer_prefix(earlier, later),
4008 "{earlier} and {later} both claim {line}:{column} in {source:?}"
4009 );
4010 }
4011 }
4012
4013 fn is_pointer_prefix(ancestor: &str, descendant: &str) -> bool {
4017 descendant
4018 .strip_prefix(ancestor)
4019 .is_some_and(|rest| ancestor.is_empty() || rest.is_empty() || rest.starts_with('/'))
4020 }
4021
4022 fn assert_written_entries_keep_anchors(
4039 source: &str,
4040 value: &serde_json::Map<String, serde_json::Value>,
4041 anchors: &FrontmatterAnchors,
4042 ) -> usize {
4043 assert_written_members_keep_anchors(source, value, &mut String::new(), anchors)
4044 }
4045
4046 fn assert_written_members_keep_anchors(
4047 source: &str,
4048 members: &serde_json::Map<String, serde_json::Value>,
4049 pointer: &mut String,
4050 anchors: &FrontmatterAnchors,
4051 ) -> usize {
4052 let mut required = 0;
4053 for (key, member) in members {
4054 let restore = pointer.len();
4055 push_pointer_token(pointer, key);
4056 if !text_may_be_textless(key) {
4057 required += 1;
4058 assert_anchor_kept(source, pointer, anchors);
4059 }
4060 required += assert_written_values_keep_anchors(source, member, pointer, anchors);
4061 pointer.truncate(restore);
4062 }
4063 required
4064 }
4065
4066 fn assert_written_values_keep_anchors(
4067 source: &str,
4068 value: &serde_json::Value,
4069 pointer: &mut String,
4070 anchors: &FrontmatterAnchors,
4071 ) -> usize {
4072 match value {
4073 serde_json::Value::Object(members) => {
4074 assert_written_members_keep_anchors(source, members, pointer, anchors)
4075 }
4076 serde_json::Value::Array(elements) => {
4077 let mut required = 0;
4078 for (index, element) in elements.iter().enumerate() {
4079 let restore = pointer.len();
4080 pointer.push('/');
4081 pointer.push_str(&index.to_string());
4082 if !value_may_be_textless(element) {
4083 required += 1;
4084 assert_anchor_kept(source, pointer, anchors);
4085 }
4086 required +=
4087 assert_written_values_keep_anchors(source, element, pointer, anchors);
4088 pointer.truncate(restore);
4089 }
4090 required
4091 }
4092 _ => 0,
4093 }
4094 }
4095
4096 fn assert_anchor_kept(source: &str, pointer: &str, anchors: &FrontmatterAnchors) {
4097 assert!(
4098 anchors.get(pointer).is_some(),
4099 "{pointer} is written but kept no anchor in {source:?}"
4100 );
4101 }
4102
4103 fn value_may_be_textless(value: &serde_json::Value) -> bool {
4105 match value {
4106 serde_json::Value::Null => true,
4107 serde_json::Value::String(text) => text_may_be_textless(text),
4108 _ => false,
4109 }
4110 }
4111
4112 fn text_may_be_textless(text: &str) -> bool {
4119 text.chars().all(|character| character == '\n')
4120 }
4121
4122 const ARBITRARY_ELEMENTS: &[&str] = &[
4138 "-",
4139 "- \"\"",
4140 "- ''",
4141 "- >-",
4142 "- |",
4143 "- |+\n",
4144 "- |+\n\n",
4145 "- null",
4146 "- ~",
4147 "- 1",
4148 "- ok",
4149 "- \" \"",
4150 "- >-\n text",
4151 "- |\n text",
4152 "- key: 1",
4153 "- [1, 2]",
4154 "- {p: 1}",
4155 "- \"\": 1",
4156 "- '': 1\n next: 2",
4157 "- {\"\": 1}",
4158 "- {'': 1, next: 2}",
4159 ];
4160
4161 const ARBITRARY_TEXTLESS_ELEMENTS: usize = 7;
4163
4164 const ARBITRARY_EMPTY_KEY_ELEMENTS: usize = 4;
4167
4168 fn holds_spelling(source: &str, spellings: &[&str]) -> bool {
4177 spellings.iter().any(|spelling| {
4178 let written = format!("\n {}\n", spelling.trim_end());
4179 source.match_indices(&written).any(|(index, matched)| {
4180 let rest = &source[index + matched.len()..];
4181 !rest.starts_with(" ") && !rest.starts_with(" : ")
4182 })
4183 })
4184 }
4185
4186 const ARBITRARY_KEYS: &[&str] = &[
4196 "? >-",
4197 "? |",
4198 "? |+\n",
4199 "? \"\"",
4200 "? ''",
4201 "? >-\n text",
4202 "? |\n text",
4203 "? \" \"",
4204 "? plain",
4205 "? plain\n : 1",
4206 "? multi\n line\n : 1",
4207 "plain: 1",
4208 "\"quoted\": 1",
4209 "'single': 1",
4210 ];
4211
4212 const ARBITRARY_TEXTLESS_KEYS: usize = 5;
4214
4215 fn arbitrary_frontmatter_document() -> impl Strategy<Value = String> {
4228 let indent = prop_oneof![9 => Just(0usize), 1 => 1usize..3];
4229 let body = prop_oneof![
4230 2 => (proptest::bool::ANY, "([a-z0-9\u{00e4}\u{00f6} ]{0,8}|[a-z0-9\u{00e4}\u{00f6}, ]{0,8}|(\r|[ ]|.){0,10})")
4232 .prop_map(|(flow, value)| if flow { format!(" [{value}]") } else { format!(" {value}") }),
4233 1 => proptest::collection::vec(0..ARBITRARY_ELEMENTS.len(), 1..5)
4235 .prop_map(|elements| {
4236 let mut text = String::new();
4237 for element in elements {
4238 text.push_str("\n ");
4239 text.push_str(ARBITRARY_ELEMENTS[element]);
4240 }
4241 text
4242 }),
4243 1 => (0..ARBITRARY_KEYS.len()).prop_map(|key| {
4247 format!("\n {}\n next: 2", ARBITRARY_KEYS[key])
4248 }),
4249 ];
4250 proptest::collection::vec(("[a-z\u{00e0}-\u{00ff}]{1,3}", indent, body), 1..6).prop_map(
4251 |entries| {
4252 let mut text = String::new();
4253 for (index, (key, indent, body)) in entries.into_iter().enumerate() {
4254 text.push_str(&" ".repeat(indent));
4255 text.push_str(&key);
4256 text.push_str(&index.to_string());
4257 text.push(':');
4258 text.push_str(&body);
4259 text.push('\n');
4260 }
4261 format!("---\n{text}---\n\n# Title\n")
4262 },
4263 )
4264 }
4265
4266 proptest! {
4267 #[test]
4268 fn arbitrary_utf8_input_is_total_and_offsets_are_valid(source in any::<String>()) {
4269 let document = parse_markdown(&source, MarkdownOptions::default());
4270 assert_valid_section_ranges(&source, &document.sections);
4271 match document.frontmatter {
4274 DocumentFrontmatter::Absent => {}
4275 DocumentFrontmatter::Mapping { location, .. }
4276 | DocumentFrontmatter::Invalid { location, .. } => {
4277 assert_valid_range(&source, location.range);
4278 prop_assert!(location.start_line >= 1);
4279 prop_assert!(location.end_line >= location.start_line);
4280 }
4281 }
4282 }
4283
4284 #[test]
4285 fn frontmatter_anchors_stay_within_their_own_line(
4286 source in arbitrary_frontmatter_document(),
4287 ) {
4288 let document = parse_markdown(&source, MarkdownOptions::default());
4289 if let DocumentFrontmatter::Mapping { location, value, anchors } = &document.frontmatter {
4290 assert_valid_anchors(&source, location, anchors);
4291 assert_written_entries_keep_anchors(&source, value, anchors);
4292 }
4293 }
4294 }
4295
4296 #[test]
4297 fn arbitrary_frontmatter_documents_reach_textless_entries() {
4298 use proptest::{strategy::ValueTree, test_runner::TestRunner};
4305
4306 const SAMPLES: usize = 512;
4307 let strategy = arbitrary_frontmatter_document();
4308 let mut runner = TestRunner::deterministic();
4309 let (mut parsed, mut sequences, mut mappings) = (0, 0, 0);
4310 let (mut textless_elements, mut textless_keys, mut required) = (0, 0, 0);
4311 let mut empty_key_elements = 0;
4312 for _ in 0..SAMPLES {
4313 let source = strategy
4314 .new_tree(&mut runner)
4315 .expect("the strategy generates a document")
4316 .current();
4317 let document = parse_markdown(&source, MarkdownOptions::default());
4318 let DocumentFrontmatter::Mapping { value, anchors, .. } = &document.frontmatter else {
4319 continue;
4320 };
4321 parsed += 1;
4322 required += assert_written_entries_keep_anchors(&source, value, anchors);
4323 let holds = |spellings: &[&str]| holds_spelling(&source, spellings);
4324 if source.contains("\n -") {
4325 sequences += 1;
4326 if holds(&ARBITRARY_ELEMENTS[..ARBITRARY_TEXTLESS_ELEMENTS]) {
4327 textless_elements += 1;
4328 }
4329 if holds(
4330 &ARBITRARY_ELEMENTS[ARBITRARY_ELEMENTS.len() - ARBITRARY_EMPTY_KEY_ELEMENTS..],
4331 ) {
4332 empty_key_elements += 1;
4333 }
4334 }
4335 if source.contains("\n next: 2") {
4336 mappings += 1;
4337 if holds(&ARBITRARY_KEYS[..ARBITRARY_TEXTLESS_KEYS]) {
4338 textless_keys += 1;
4339 }
4340 }
4341 }
4342 println!(
4343 "of {SAMPLES} generated documents: {parsed} parsed as a mapping, \
4344 {sequences} held a block sequence ({textless_elements} of them a textless \
4345 element, {empty_key_elements} of them a mapping under a quoted empty key), \
4346 {mappings} held a nested mapping ({textless_keys} of them a \
4347 textless key); {required} written entries had to keep an anchor"
4348 );
4349
4350 assert!(parsed >= SAMPLES / 4, "only {parsed} documents parsed");
4351 assert!(
4352 sequences >= SAMPLES / 16,
4353 "only {sequences} documents held a block sequence"
4354 );
4355 assert!(
4356 textless_elements >= SAMPLES / 32,
4357 "only {textless_elements} documents held a textless element"
4358 );
4359 assert!(
4364 empty_key_elements >= SAMPLES / 32,
4365 "only {empty_key_elements} documents held a mapping under a quoted empty key"
4366 );
4367 assert!(
4368 mappings >= SAMPLES / 16,
4369 "only {mappings} documents held a nested mapping"
4370 );
4371 assert!(
4372 textless_keys >= SAMPLES / 32,
4373 "only {textless_keys} documents held a textless key"
4374 );
4375 assert!(
4376 required >= SAMPLES,
4377 "only {required} written entries were required to keep an anchor"
4378 );
4379 }
4380}