satteri_pulldown_cmark/
utils.rs

1//! Miscellaneous utilities to increase comfort.
2//! Special thanks to:
3//!
4//! - <https://github.com/BenjaminRi/Redwood-Wiki/blob/master/src/markdown_utils.rs>.
5//!   Its author authorized the use of this GPL code in this project in
6//!   <https://github.com/raphlinus/pulldown-cmark/issues/507>.
7//!
8//! - <https://gist.github.com/rambip/a507c312ed61c99c24b2a54f98325721>.
9//!   Its author proposed the solution in
10//!   <https://github.com/raphlinus/pulldown-cmark/issues/708>.
11
12use alloc::borrow::Cow;
13use alloc::string::String;
14use core::ops::Range;
15
16use crate::{CowStr, Event};
17
18/// The source the parser works on: micromark's preprocessor drops a leading
19/// BOM before positions exist, so it is outside the position space.
20#[must_use]
21pub fn strip_leading_bom(source: &str) -> &str {
22    source.strip_prefix('\u{feff}').unwrap_or(source)
23}
24
25/// Decode HTML5 character references (`&gt;`, `&amp;`, `&#x3C;`, `&#123;`, …)
26/// inside a string. Unrecognised `&foo` runs are left as-is.
27///
28/// JSX text and JSX literal attribute values both go through HTML entity
29/// decoding before reaching the runtime: `<p>&gt;</p>` and `<p title="&gt;"/>`
30/// both materialise as a `>` character. This helper is shared so the two
31/// call sites agree on the entity table.
32pub fn decode_html_entities(s: &str) -> Cow<'_, str> {
33    if !s.contains('&') {
34        return Cow::Borrowed(s);
35    }
36    let bytes = s.as_bytes();
37    let mut out = String::with_capacity(s.len());
38    let mut i = 0;
39    while i < bytes.len() {
40        if bytes[i] == b'&' {
41            let (consumed, replacement) = crate::scanners::scan_entity(&bytes[i..]);
42            if consumed > 0 {
43                if let Some(rep) = replacement {
44                    out.push_str(&rep);
45                }
46                i += consumed;
47                continue;
48            }
49        }
50        let b = bytes[i];
51        let ch_len = if b < 0xC0 {
52            1
53        } else if b < 0xE0 {
54            2
55        } else if b < 0xF0 {
56            3
57        } else {
58            4
59        };
60        out.push_str(&s[i..i + ch_len]);
61        i += ch_len;
62    }
63    Cow::Owned(out)
64}
65
66/// Merge consecutive `Event::Text` events into only one.
67#[derive(Debug)]
68pub struct TextMergeStream<'a, I> {
69    inner: TextMergeWithOffset<'a, DummyOffsets<I>>,
70}
71
72impl<'a, I> TextMergeStream<'a, I>
73where
74    I: Iterator<Item = Event<'a>>,
75{
76    pub fn new(iter: I) -> Self {
77        Self {
78            inner: TextMergeWithOffset::new(DummyOffsets(iter)),
79        }
80    }
81}
82
83impl<'a, I> Iterator for TextMergeStream<'a, I>
84where
85    I: Iterator<Item = Event<'a>>,
86{
87    type Item = Event<'a>;
88
89    fn next(&mut self) -> Option<Self::Item> {
90        self.inner.next().map(|(event, _)| event)
91    }
92}
93
94#[derive(Debug)]
95struct DummyOffsets<I>(I);
96
97impl<'a, I> Iterator for DummyOffsets<I>
98where
99    I: Iterator<Item = Event<'a>>,
100{
101    type Item = (Event<'a>, Range<usize>);
102
103    fn next(&mut self) -> Option<Self::Item> {
104        self.0.next().map(|event| (event, 0..0))
105    }
106}
107
108/// Merge consecutive `Event::Text` events into only one, with offsets.
109///
110/// Compatible with with [`OffsetIter`](crate::OffsetIter).
111#[derive(Debug)]
112pub struct TextMergeWithOffset<'a, I> {
113    iter: I,
114    last_event: Option<(Event<'a>, Range<usize>)>,
115}
116
117impl<'a, I> TextMergeWithOffset<'a, I>
118where
119    I: Iterator<Item = (Event<'a>, Range<usize>)>,
120{
121    pub fn new(iter: I) -> Self {
122        Self {
123            iter,
124            last_event: None,
125        }
126    }
127
128    /// Access the inner iterator (e.g. to retrieve parser state after iteration).
129    pub fn inner(&self) -> &I {
130        &self.iter
131    }
132}
133
134impl<'a, I> Iterator for TextMergeWithOffset<'a, I>
135where
136    I: Iterator<Item = (Event<'a>, Range<usize>)>,
137{
138    type Item = (Event<'a>, Range<usize>);
139
140    fn next(&mut self) -> Option<Self::Item> {
141        match (self.last_event.take(), self.iter.next()) {
142            (
143                Some((Event::Text(last_text), last_offset)),
144                Some((Event::Text(next_text), next_offset)),
145            ) => {
146                // We need to start merging consecutive text events together into one
147                let mut string_buf: String = last_text.into_string();
148                string_buf.push_str(&next_text);
149                let mut offset = last_offset;
150                offset.end = next_offset.end;
151                loop {
152                    // Avoid recursion to avoid stack overflow and to optimize concatenation
153                    match self.iter.next() {
154                        Some((Event::Text(next_text), next_offset)) => {
155                            string_buf.push_str(&next_text);
156                            offset.end = next_offset.end;
157                        }
158                        next_event => {
159                            self.last_event = next_event;
160                            if string_buf.is_empty() {
161                                // Discard text event(s) altogether if there is no text
162                                break self.next();
163                            } else {
164                                break Some((
165                                    Event::Text(CowStr::Boxed(string_buf.into_boxed_str())),
166                                    offset,
167                                ));
168                            }
169                        }
170                    }
171                }
172            }
173            (None, Some(next_event)) => {
174                // This only happens once during the first iteration and if there are items
175                self.last_event = Some(next_event);
176                self.next()
177            }
178            (None, None) => {
179                // This happens when the iterator is depleted
180                None
181            }
182            (last_event, next_event) => {
183                // The ordinary case, emit one event after the other without modification
184                self.last_event = next_event;
185                last_event
186            }
187        }
188    }
189}
190
191#[cfg(test)]
192mod test {
193    use alloc::vec::Vec;
194
195    use super::*;
196    use crate::Parser;
197
198    #[test]
199    fn text_merge_stream_indent() {
200        let source = r#"
201    first line
202    second line
203"#;
204        let parser = TextMergeStream::new(Parser::new(source));
205        let text_events: Vec<_> = parser.filter(|e| matches!(e, Event::Text(_))).collect();
206        assert_eq!(
207            text_events,
208            [Event::Text("first line\nsecond line\n".into())]
209        );
210    }
211
212    #[test]
213    fn text_merge_with_offset_indent() {
214        let source = r#"
215    first line
216    second line
217"#;
218        let parser = TextMergeWithOffset::new(Parser::new(source).into_offset_iter());
219        let text_events: Vec<_> = parser
220            .filter(|e| matches!(e, (Event::Text(_), _)))
221            .collect();
222        assert_eq!(
223            text_events,
224            [(Event::Text("first line\nsecond line\n".into()), 5..32)]
225        );
226    }
227
228    #[test]
229    fn text_merge_empty_is_discarded() {
230        let events = [
231            Event::Rule,
232            Event::Text("".into()),
233            Event::Text("".into()),
234            Event::Rule,
235        ];
236        let result: Vec<_> = TextMergeStream::new(events.into_iter()).collect();
237        assert_eq!(result, [Event::Rule, Event::Rule]);
238    }
239}
satteri_pulldown_cmark/utils.rs

satteri_pulldown_cmark/
utils.rs
