atuin_common/string/
highlighted.rs

1//! Provides the [`HighlightedText`] utility which expresses the idea of a piece of text with
2//! highlight markers.
3//!
4//! Consider the string:
5//!
6//! ```text
7//! h e l l o w o r l d
8//! ```
9//!
10//! If you add some bytes which express highlighting zones, you can end up with a string
11//!
12//! ```text
13//! [ h e l ] l o w [ o r ] l d
14//! ```
15//!
16//! In this example, I use the bytes `[` and `]` for visual clarity, but normally you'd use unicode
17//! characters.
18//!
19//! We consider the text `h e l` and `[ o r ]` to be "highlighted".
20//!
21//! TODO(markovejnovic): Currently, only [`char`]s are supported as markers, but it would perhaps be
22//!                      wise to support arbitrary marker strings. This is blocked by using smolstr
23//!                      or a similar small-string-optimized utility.
24//!
25//! TODO(markovejnovic): Another relatively useful feature here would be to add support for multiple
26//!                      different highlighting schemes -- nested highlights, as well as
27//!                      cross-region highlights. We don't need this at this moment, but it would be
28//!                      good to support highlighting `[ h ( e l ) ] o w o (r l) d`, as well as
29//!                      `[ h ( e l ] o w ) o ( r l ) d`.
30use std::borrow::Cow;
31use std::fmt;
32use std::fmt::Write as _;
33use std::ops::Range;
34
35use thiserror::Error;
36
37#[derive(Debug, Error)]
38pub enum NewTextHighlighterError {
39    #[error("identical start and end markers given: {0:?}")]
40    IdenticalMarkers([char; 2]),
41}
42
43/// Structure which encodes how exactly text was highlighted.
44#[derive(Debug, Clone, Copy)]
45pub struct TextHighlighter {
46    /// The first marker used to highlight a piece of text.
47    open: char,
48    /// The second marker used to highlight a piece of text.
49    close: char,
50}
51
52impl Default for TextHighlighter {
53    fn default() -> Self {
54        // Private-use codepoints, so they never collide with anything a terminal would show.
55        const DEFAULT_MATCH_OPEN: char = '\u{E000}';
56        const DEFAULT_MATCH_CLOSE: char = '\u{E001}';
57
58        Self::with_markers([DEFAULT_MATCH_OPEN, DEFAULT_MATCH_CLOSE])
59            .expect("the default markers are distinct")
60    }
61}
62
63impl TextHighlighter {
64    /// Create the highlighter with the given markers.
65    pub fn with_markers(markers: [char; 2]) -> Result<Self, NewTextHighlighterError> {
66        if markers[0] == markers[1] {
67            return Err(NewTextHighlighterError::IdenticalMarkers(markers));
68        }
69
70        Ok(Self {
71            open: markers[0],
72            close: markers[1],
73        })
74    }
75
76    /// The `[open, close]` markers this highlighter wraps matches in.
77    #[must_use]
78    pub fn markers(self) -> [char; 2] {
79        [self.open, self.close]
80    }
81
82    /// Take a string which may or may not contain highlight markers and strip it of said highlight
83    /// markers.
84    ///
85    /// TODO(markovejnovic): Could be more optimized for mutable strings and sanitize them in-place.
86    #[must_use]
87    pub fn sanitize(self, text: &str) -> Cow<'_, str> {
88        if text.contains([self.open, self.close]) {
89            Cow::Owned(text.replace([self.open, self.close], ""))
90        } else {
91            Cow::Borrowed(text)
92        }
93    }
94
95    /// Take a string which may or may not contain highlight markers and mark it as a highlighted
96    /// string.
97    ///
98    /// If the string does not contain any highlight markers, the resulting HighlightedText won't
99    /// either.
100    pub fn as_highlighted<S: AsRef<str>>(self, data: S) -> HighlightedText<S> {
101        HighlightedText {
102            data,
103            highlighter: self,
104        }
105    }
106}
107
108/// Represents text which was highlighted by the `TextHighlighter`.
109///
110/// `Display` implementations come in the form of [`Self::display_plain`], [`Self::display_subs`]
111/// and [`Self::display_raw`].
112#[derive(Clone, Copy)]
113pub struct HighlightedText<S> {
114    data: S,
115    highlighter: TextHighlighter,
116}
117
118impl<S> HighlightedText<S> {
119    /// Grab a handle to the raw data.
120    pub fn raw(&self) -> &S {
121        &self.data
122    }
123
124    /// Grab a mutable handle to the underlying data.
125    pub fn raw_mut(&mut self) -> &mut S {
126        &mut self.data
127    }
128
129    pub fn markers(&self) -> [char; 2] {
130        self.highlighter.markers()
131    }
132
133    /// Swap the underlying string type, keeping the highlighter (e.g. `line.map(str::to_owned)`).
134    pub fn map<T>(self, f: impl FnOnce(S) -> T) -> HighlightedText<T> {
135        HighlightedText {
136            data: f(self.data),
137            highlighter: self.highlighter,
138        }
139    }
140}
141
142impl<S: AsRef<str>> AsRef<str> for HighlightedText<S> {
143    fn as_ref(&self) -> &str {
144        self.data.as_ref()
145    }
146}
147
148impl<S: AsRef<str>> fmt::Debug for HighlightedText<S> {
149    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
150        write!(f, "{:?}", self.data.as_ref())
151    }
152}
153
154impl<S: AsRef<str>> HighlightedText<S> {
155    pub fn ranges(&self) -> impl Iterator<Item = Range<usize>> + '_ {
156        HighlightedRanges {
157            src: self,
158            cursor: 0,
159            start: None,
160        }
161    }
162
163    /// Walk the text as a stream of marker-free chunks: [`Piece::Text`] for unhighlighted runs and
164    /// [`Piece::Match`] for each highlighted span.
165    ///
166    /// Concatenating every chunk yields the same string as [`Self::display_plain`], and the
167    /// [`Piece::Match`] chunks are exactly the spans [`Self::ranges`] reports. Empty chunks are
168    /// skipped.
169    pub fn pieces(&self) -> impl Iterator<Item = Piece<'_>> + '_ {
170        Pieces {
171            text: self.data.as_ref(),
172            open: self.highlighter.open,
173            close: self.highlighter.close,
174            ranges: HighlightedRanges {
175                src: self,
176                cursor: 0,
177                start: None,
178            },
179            cursor: 0,
180            gap: "",
181            pending: None,
182            tail_done: false,
183        }
184    }
185
186    /// Whether any highlighted span is present.
187    pub fn has_match(&self) -> bool {
188        self.ranges().next().is_some()
189    }
190
191    /// Each line as its own highlighted text. A match never spans a newline, so every span stays
192    /// within one line.
193    pub fn lines(&self) -> impl Iterator<Item = HighlightedText<&str>> + '_ {
194        self.data.as_ref().lines().map(|line| self.highlighter.as_highlighted(line))
195    }
196
197    /// The marker-free text together with the byte range of every match within *that* text --
198    /// unlike [`Self::ranges`], whose offsets index the raw, marker-bearing string.
199    pub fn to_plain(&self) -> Plain<'_> {
200        let raw = self.data.as_ref();
201        if !raw.contains(self.highlighter.markers()) {
202            return Plain {
203                text: Cow::Borrowed(raw),
204                ranges: Vec::new(),
205            };
206        }
207        let (text, ranges) =
208            self.pieces().fold((String::new(), Vec::new()), |(mut plain, mut ranges), piece| {
209                let start = plain.len();
210                match piece {
211                    Piece::Text(text) => plain.push_str(text),
212                    Piece::Match(text) => {
213                        plain.push_str(text);
214                        ranges.push(start..plain.len());
215                    }
216                }
217                (plain, ranges)
218            });
219        Plain {
220            text: Cow::Owned(text),
221            ranges,
222        }
223    }
224
225    /// `Display` the highlighted text, stripping away the highlight markers.
226    pub fn display_plain(&self) -> impl fmt::Display + '_ {
227        DisplayPlain(self)
228    }
229
230    /// `Display` the highlighted text, replacing highlighted markers with `subs`.
231    pub fn display_subs(&self, subs: [char; 2]) -> impl fmt::Display + '_ {
232        DisplaySubs { src: self, subs }
233    }
234
235    /// `Display` the highlighted text as-is, including markers.
236    pub fn display_raw(&self) -> impl fmt::Display + '_ {
237        DisplayRaw(self)
238    }
239}
240
241/// The marker-free view of a [`HighlightedText`], from [`HighlightedText::to_plain`].
242#[derive(Debug, Clone, PartialEq, Eq)]
243pub struct Plain<'a> {
244    /// Borrowed from the source when it holds no markers.
245    pub text: Cow<'a, str>,
246    /// Byte ranges of the matches within `text`.
247    pub ranges: Vec<Range<usize>>,
248}
249
250/// One marker-free chunk of a [`HighlightedText`], produced by [`HighlightedText::pieces`].
251#[derive(Debug, Clone, Copy, PartialEq, Eq)]
252pub enum Piece<'a> {
253    /// A run of text outside any highlight.
254    Text(&'a str),
255    /// The content of one highlighted match.
256    Match(&'a str),
257}
258
259struct Pieces<'a, S> {
260    text: &'a str,
261    open: char,
262    close: char,
263    ranges: HighlightedRanges<'a, S>,
264    /// Start of the not-yet-emitted region of `text`, just past the previous match's close marker.
265    cursor: usize,
266    /// The gap before `pending`, drained into marker-free [`Piece::Text`] runs before it is emitted.
267    gap: &'a str,
268    /// A match whose content is emitted once `gap` is drained.
269    pending: Option<&'a str>,
270    /// Whether the trailing gap after the final match has been queued into `gap`.
271    tail_done: bool,
272}
273
274impl<'a, S: AsRef<str>> Iterator for Pieces<'a, S> {
275    type Item = Piece<'a>;
276
277    fn next(&mut self) -> Option<Piece<'a>> {
278        loop {
279            // Drain the current gap into marker-free text runs, dropping any stray markers.
280            if !self.gap.is_empty() {
281                let Some(at) = self.gap.find([self.open, self.close]) else {
282                    return Some(Piece::Text(std::mem::take(&mut self.gap)));
283                };
284                let seg = &self.gap[..at];
285                let marker_len = if self.gap[at..].starts_with(self.open) {
286                    self.open.len_utf8()
287                } else {
288                    self.close.len_utf8()
289                };
290                self.gap = &self.gap[at + marker_len..];
291                if !seg.is_empty() {
292                    return Some(Piece::Text(seg));
293                }
294                continue;
295            }
296
297            // Gap drained: emit the pending match, then advance to the next span.
298            if let Some(matched) = self.pending.take() {
299                if !matched.is_empty() {
300                    return Some(Piece::Match(matched));
301                }
302                continue;
303            }
304
305            match self.ranges.next() {
306                Some(r) => {
307                    let open_start = (r.start - self.open.len_utf8()).max(self.cursor);
308                    let next_cursor = r.end + self.close.len_utf8();
309                    self.gap = &self.text[self.cursor..open_start];
310                    self.pending = Some(&self.text[r]);
311                    self.cursor = next_cursor;
312                }
313                None if !self.tail_done => {
314                    self.tail_done = true;
315                    self.gap = &self.text[self.cursor..];
316                }
317                None => return None,
318            }
319        }
320    }
321}
322
323struct HighlightedRanges<'a, S> {
324    src: &'a HighlightedText<S>,
325    cursor: usize,
326    start: Option<usize>,
327}
328
329impl<S: AsRef<str>> Iterator for HighlightedRanges<'_, S> {
330    type Item = Range<usize>;
331
332    fn next(&mut self) -> Option<Range<usize>> {
333        let text = self.src.data.as_ref();
334        let open = self.src.highlighter.open;
335        let close = self.src.highlighter.close;
336        loop {
337            let rest = &text[self.cursor..];
338            let at = rest.find([open, close])?;
339            let marker_pos = self.cursor + at;
340            if rest[at..].starts_with(open) {
341                self.cursor = marker_pos + open.len_utf8();
342                self.start = Some(self.cursor);
343            } else {
344                self.cursor = marker_pos + close.len_utf8();
345                if let Some(start) = self.start.take() {
346                    return Some(start..marker_pos);
347                }
348            }
349        }
350    }
351}
352
353struct DisplayPlain<'a, S>(&'a HighlightedText<S>);
354
355impl<S: AsRef<str>> fmt::Display for DisplayPlain<'_, S> {
356    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
357        let h = self.0.highlighter;
358        for piece in self.0.data.as_ref().split([h.open, h.close]) {
359            f.write_str(piece)?;
360        }
361        Ok(())
362    }
363}
364
365struct DisplaySubs<'a, S> {
366    src: &'a HighlightedText<S>,
367    subs: [char; 2],
368}
369
370impl<S: AsRef<str>> fmt::Display for DisplaySubs<'_, S> {
371    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
372        let h = self.src.highlighter;
373        let [open_sub, close_sub] = self.subs;
374        for c in self.src.data.as_ref().chars() {
375            if c == h.open {
376                f.write_char(open_sub)?;
377            } else if c == h.close {
378                f.write_char(close_sub)?;
379            } else {
380                f.write_char(c)?;
381            }
382        }
383        Ok(())
384    }
385}
386
387struct DisplayRaw<'a, S>(&'a HighlightedText<S>);
388
389impl<S: AsRef<str>> fmt::Display for DisplayRaw<'_, S> {
390    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
391        f.write_str(self.0.data.as_ref())
392    }
393}
394
395pub type HighlightedString = HighlightedText<String>;
396pub type HighlightedStr<'a> = HighlightedText<&'a str>;
397pub type HighlightedCowStr<'a> = HighlightedText<Cow<'a, str>>;
398
399#[cfg(feature = "proto")]
400mod proto {
401    use thiserror::Error;
402
403    use super::{HighlightedString, HighlightedText, NewTextHighlighterError, TextHighlighter};
404
405    #[derive(Clone, PartialEq, Eq, Hash, prost::Message)]
406    pub struct HighlightedTextProto {
407        #[prost(uint32, tag = "1")]
408        pub open: u32,
409        #[prost(uint32, tag = "2")]
410        pub close: u32,
411        #[prost(string, tag = "3")]
412        pub raw: String,
413    }
414
415    #[derive(Debug, Error)]
416    pub enum FromHighlightedTextProtoError {
417        #[error("marker code point {0:#x} is not a valid char")]
418        InvalidMarker(u32),
419        #[error(transparent)]
420        Markers(#[from] NewTextHighlighterError),
421    }
422
423    impl<S: AsRef<str>> From<&HighlightedText<S>> for HighlightedTextProto {
424        fn from(value: &HighlightedText<S>) -> Self {
425            let [open, close] = value.highlighter.markers();
426            Self {
427                open: u32::from(open),
428                close: u32::from(close),
429                raw: value.data.as_ref().to_owned(),
430            }
431        }
432    }
433
434    impl TryFrom<HighlightedTextProto> for HighlightedString {
435        type Error = FromHighlightedTextProtoError;
436
437        fn try_from(value: HighlightedTextProto) -> Result<Self, Self::Error> {
438            let open = char::from_u32(value.open)
439                .ok_or(FromHighlightedTextProtoError::InvalidMarker(value.open))?;
440            let close = char::from_u32(value.close)
441                .ok_or(FromHighlightedTextProtoError::InvalidMarker(value.close))?;
442            Ok(TextHighlighter::with_markers([open, close])?.as_highlighted(value.raw))
443        }
444    }
445}
446
447#[cfg(feature = "proto")]
448pub use proto::{FromHighlightedTextProtoError, HighlightedTextProto};
449
450#[cfg(test)]
451mod tests {
452    use pretty_assertions::assert_eq;
453    use proptest::prelude::*;
454    use rstest::rstest;
455
456    use super::*;
457
458    /// A highlighter with visible, multibyte markers — distinct from anything the test bodies hold.
459    fn highlighter() -> TextHighlighter {
460        TextHighlighter::with_markers(['«', '»']).expect("distinct markers")
461    }
462
463    /// Assert `ranges()` yields `expected`, and that indexing the raw (still-marked) text with each
464    /// range recovers `hits`.
465    fn assert_ranges(h: TextHighlighter, body: &str, expected: &[Range<usize>], hits: &[&str]) {
466        let hl = h.as_highlighted(body);
467        let ranges: Vec<Range<usize>> = hl.ranges().collect();
468        assert_eq!(ranges.as_slice(), expected);
469        let raw: &str = hl.as_ref();
470        let got: Vec<&str> = ranges.iter().map(|r| &raw[r.clone()]).collect();
471        assert_eq!(got.as_slice(), hits);
472    }
473
474    #[rstest]
475    #[case::ascii(['x', 'x'], false)]
476    #[case::emoji(['😀', '😀'], false)]
477    #[case::whitespace([' ', ' '], false)]
478    #[case::null(['\0', '\0'], false)]
479    #[case::brackets(['[', ']'], true)]
480    #[case::mixed_whitespace([' ', '\n'], true)]
481    #[case::control(['\0', '\u{1}'], true)]
482    #[case::default_pua(['\u{E000}', '\u{E001}'], true)]
483    fn with_markers_ok_iff_distinct(#[case] markers: [char; 2], #[case] ok: bool) {
484        assert_eq!(TextHighlighter::with_markers(markers).is_ok(), ok);
485    }
486
487    #[test]
488    fn identical_markers_error_carries_the_markers() {
489        assert!(matches!(
490            TextHighlighter::with_markers(['x', 'x']),
491            Err(NewTextHighlighterError::IdenticalMarkers(['x', 'x']))
492        ));
493    }
494
495    #[rstest]
496    #[case::empty("", "", true)]
497    #[case::clean("clean text", "clean text", true)]
498    #[case::only_open("a«b«c", "abc", false)]
499    #[case::only_close("a»b»c", "abc", false)]
500    #[case::both_markers("a«b»c«d»", "abcd", false)]
501    // Entirely-markers input still allocates: same content as `sanitize("")` but a different Cow.
502    #[case::all_markers("«»«»", "", false)]
503    fn sanitize_strips_markers_and_borrows_only_when_clean(
504        #[case] input: &str,
505        #[case] expected: &str,
506        #[case] borrowed: bool,
507    ) {
508        let out = highlighter().sanitize(input);
509        assert_eq!(matches!(out, Cow::Borrowed(_)), borrowed);
510        assert_eq!(out, expected);
511    }
512
513    #[test]
514    fn markers_chosen_from_ordinary_text_delete_that_text() {
515        // `with_markers` only rejects markers equal to each other, never ones that collide with
516        // content — so visible markers turn `sanitize` into a data-destroying filter. This is why
517        // `Default` uses private-use codepoints.
518        let h = TextHighlighter::with_markers(['a', 'b']).unwrap();
519        assert_eq!(h.sanitize("banana bread"), "nn red");
520    }
521
522    // `ranges()` indexes the raw (still-marked) text, pointing at the content between each
523    // open/close pair.
524    #[rstest]
525    #[case::single_span("the build «failed» now", vec![12..18], vec!["failed"])]
526    #[case::each_span_its_own_range("«a» b «c»", vec![2..3, 10..11], vec!["a", "c"])]
527    #[case::multibyte_prefix_keeps_offsets("café «error»", vec![8..13], vec!["error"])]
528    #[case::no_markers("nothing here", vec![], vec![])]
529    // A second open before a close overwrites the first, so the outer span is silently dropped.
530    #[case::nested_open_keeps_inner("«a«b»", vec![5..6], vec!["b"])]
531    #[case::nested_pair_keeps_inner("«a«b»c»", vec![5..6], vec!["b"])]
532    #[case::doubled_markers_collapse("««x»»", vec![4..5], vec!["x"])]
533    // An open immediately closed yields a valid zero-width range.
534    #[case::empty_span("«»", vec![2..2], vec![""])]
535    #[case::adjacent_empty_spans("«»«»", vec![2..2, 6..6], vec!["", ""])]
536    #[case::empty_span_between_real("«a»«»«b»", vec![2..3, 7..7, 11..12], vec!["a", "", "b"])]
537    #[case::back_to_back_spans("«a»«b»", vec![2..3, 7..8], vec!["a", "b"])]
538    #[case::whitespace_span("«   »", vec![2..5], vec!["   "])]
539    // Unmatched opens never emit; an orphan close is a no-op.
540    #[case::unterminated_opens("«a«b", vec![], vec![])]
541    #[case::orphan_close_then_open("»«", vec![], vec![])]
542    #[case::orphan_close_ignored("a»b«c»", vec![6..7], vec!["c"])]
543    #[case::extra_close_ignored("«a»b»c", vec![2..3], vec!["a"])]
544    fn ranges_index_the_raw_text(
545        #[case] body: &str,
546        #[case] expected: Vec<Range<usize>>,
547        #[case] hits: Vec<&str>,
548    ) {
549        assert_ranges(highlighter(), body, &expected, &hits);
550    }
551
552    #[rstest]
553    #[case::multibyte_text_before("café \u{E000}error\u{E001}", vec![9..14], vec!["error"])]
554    #[case::multibyte_text_inside("\u{E000}café\u{E001}", vec![3..8], vec!["café"])]
555    // Ranges are code-point boundaries, not grapheme boundaries: a marker mid-grapheme splits it.
556    #[case::splits_combining_mark("e\u{E000}\u{0301}\u{E001}", vec![4..6], vec!["\u{0301}"])]
557    fn ranges_with_default_multibyte_markers(
558        #[case] body: &str,
559        #[case] expected: Vec<Range<usize>>,
560        #[case] hits: Vec<&str>,
561    ) {
562        assert_ranges(TextHighlighter::default(), body, &expected, &hits);
563    }
564
565    #[test]
566    fn markers_are_positional_so_swapping_them_inverts_parsing() {
567        let swapped = TextHighlighter::with_markers(['»', '«']).unwrap();
568        assert_eq!(swapped.as_highlighted("«failed»").ranges().count(), 0);
569    }
570
571    // The `display_*` views are context-free per-marker transforms: every marker char is acted on
572    // whether or not it pairs up, so orphan and nested markers get substituted/stripped just like
573    // matched ones. This is deliberately unlike `ranges()`, which reports only matched pairs — so
574    // no marker (a private-use codepoint by default) can ever leak into the rendered output.
575    #[rstest]
576    #[case::clean("clean text", "clean text")]
577    #[case::single_span("the build «failed» now", "the build [failed] now")]
578    #[case::two_spans("«a» b «c»", "[a] b [c]")]
579    #[case::orphan_open("«a", "[a")]
580    #[case::orphan_close("a»", "a]")]
581    #[case::nested_open("«a«b»", "[a[b]")]
582    #[case::doubled("««x»»", "[[x]]")]
583    #[case::empty_span("«»", "[]")]
584    #[case::multibyte_text("café «error»", "café [error]")]
585    fn display_subs_substitutes_every_marker(#[case] body: &str, #[case] expected: &str) {
586        let out = highlighter().as_highlighted(body).display_subs(['[', ']']).to_string();
587        assert_eq!(out, expected);
588    }
589
590    #[rstest]
591    #[case::clean("clean text", "clean text")]
592    #[case::single_span("the build «failed» now", "the build failed now")]
593    #[case::orphan_and_nested("«a«b»", "ab")]
594    #[case::doubled("««x»»", "x")]
595    fn display_plain_strips_every_marker(#[case] body: &str, #[case] expected: &str) {
596        let out = highlighter().as_highlighted(body).display_plain().to_string();
597        assert_eq!(out, expected);
598    }
599
600    #[rstest]
601    #[case::clean("clean text", "clean text", vec![])]
602    #[case::span_between_text("the «build» failed", "the build failed", vec![4..9])]
603    #[case::back_to_back("«a»«b»", "ab", vec![0..1, 1..2])]
604    #[case::multibyte_before_span("héllo «wörld»", "héllo wörld", vec![7..13])]
605    fn to_plain_indexes_the_plain_text(
606        #[case] body: &str,
607        #[case] plain: &str,
608        #[case] ranges: Vec<Range<usize>>,
609    ) {
610        let hl = highlighter().as_highlighted(body);
611        let got = hl.to_plain();
612        assert_eq!(got.text, plain);
613        assert_eq!(got.ranges, ranges);
614        assert_eq!(matches!(got.text, Cow::Borrowed(_)), ranges.is_empty());
615        for r in got.ranges {
616            assert!(got.text.is_char_boundary(r.start) && got.text.is_char_boundary(r.end));
617        }
618    }
619
620    #[test]
621    fn display_raw_is_verbatim() {
622        let body = "«a«b»";
623        assert_eq!(highlighter().as_highlighted(body).display_raw().to_string(), body);
624    }
625
626    #[rstest]
627    #[case::clean("clean text", vec![(false, "clean text")])]
628    #[case::span_between_text("the «build» failed", vec![(false, "the "), (true, "build"), (false, " failed")])]
629    #[case::back_to_back("«a»«b»", vec![(true, "a"), (true, "b")])]
630    #[case::no_markers("", vec![])]
631    #[case::empty_span_skipped("«»", vec![])]
632    // Markers are stripped context-free (like `display_plain`), but only paired spans are matches
633    // (like `ranges`): the abandoned outer open's content is plain text, only the inner span matches.
634    #[case::nested_open_outer_is_text("«a«b»", vec![(false, "a"), (true, "b")])]
635    #[case::orphan_close_stripped("»a«c»", vec![(false, "a"), (true, "c")])]
636    fn pieces_split_text_and_matches(#[case] body: &str, #[case] expected: Vec<(bool, &str)>) {
637        let hl = highlighter().as_highlighted(body);
638        let got: Vec<(bool, &str)> = hl
639            .pieces()
640            .map(|p| match p {
641                Piece::Text(t) => (false, t),
642                Piece::Match(m) => (true, m),
643            })
644            .collect();
645        assert_eq!(got, expected);
646    }
647
648    #[rstest]
649    #[case::single("a «b»", vec![("a b", true)])]
650    #[case::mixed("x\n«hit» here\ny\n", vec![("x", false), ("hit here", true), ("y", false)])]
651    #[case::blank_lines("\n\n«a»", vec![("", false), ("", false), ("a", true)])]
652    #[case::empty("", vec![])]
653    fn lines_keep_each_lines_own_markers(#[case] body: &str, #[case] expected: Vec<(&str, bool)>) {
654        let hl = highlighter().as_highlighted(body);
655        let got: Vec<(String, bool)> =
656            hl.lines().map(|l| (l.display_plain().to_string(), l.has_match())).collect();
657        let expected: Vec<(String, bool)> =
658            expected.into_iter().map(|(t, m)| (t.to_owned(), m)).collect();
659        assert_eq!(got, expected);
660    }
661
662    /// Alphabet that stresses the byte-offset arithmetic: plain ASCII for clean runs plus the two
663    /// visible markers, the private-use defaults, and a spread of multi-byte / zero-width /
664    /// combining / RTL code points so marker collisions and mid-grapheme splits are frequent.
665    fn nasty_char() -> impl Strategy<Value = char> {
666        prop_oneof![
667            3 => prop::char::range('a', 'e'),
668            2 => Just('«'),
669            2 => Just('»'),
670            1 => Just('\u{E000}'),
671            1 => Just('\u{E001}'),
672            1 => Just('é'),
673            1 => Just('中'),
674            1 => Just('👩'),
675            1 => Just('\u{200D}'),
676            1 => Just('\u{0301}'),
677            1 => Just('\u{200F}'),
678            1 => Just(' '),
679        ]
680    }
681
682    fn nasty_string() -> impl Strategy<Value = String> {
683        prop::collection::vec(nasty_char(), 0..24).prop_map(|cs| cs.into_iter().collect())
684    }
685
686    /// Two distinct markers drawn from the same nasty alphabet, so the byte width of the markers
687    /// varies against the text and can collide with the content.
688    fn distinct_markers() -> impl Strategy<Value = [char; 2]> {
689        [nasty_char(), nasty_char()].prop_filter("markers must differ", |m| m[0] != m[1])
690    }
691
692    /// Two distinct markers, text free of them, and ascending non-overlapping char-index spans over
693    /// that text. Drawing the markers from the same alphabet varies their byte width (1–4 bytes), so
694    /// the round-trip stresses the `marker.len_utf8()` cursor arithmetic against exact recovery.
695    fn markers_clean_and_spans()
696    -> impl Strategy<Value = ([char; 2], Vec<char>, Vec<(usize, usize)>)> {
697        distinct_markers().prop_flat_map(|markers| {
698            let [open, close] = markers;
699            prop::collection::vec(
700                nasty_char()
701                    .prop_filter("clean text holds no marker", move |c| *c != open && *c != close),
702                0..12,
703            )
704            .prop_flat_map(move |chars| {
705                let n = chars.len();
706                let spans = prop::collection::vec(0usize..=n, 0..8).prop_map(|mut v| {
707                    v.sort_unstable();
708                    if v.len() % 2 == 1 {
709                        v.pop();
710                    }
711                    v.as_chunks::<2>().0.iter().map(|c| (c[0], c[1])).collect::<Vec<_>>()
712                });
713                (Just(markers), Just(chars), spans)
714            })
715        })
716    }
717
718    /// Wrap each `(start_char, end_char)` span of `chars` in `markers`, returning the marked string
719    /// and the byte ranges the highlighted content occupies *within that marked string*.
720    fn wrap_spans(
721        markers: [char; 2],
722        chars: &[char],
723        spans: &[(usize, usize)],
724    ) -> (String, Vec<Range<usize>>) {
725        let [open, close] = markers;
726        let mut marked = String::new();
727        let mut ranges = Vec::with_capacity(spans.len());
728        let mut cursor = 0;
729        for &(s, e) in spans {
730            marked.extend(chars[cursor..s].iter());
731            marked.push(open);
732            let start = marked.len();
733            marked.extend(chars[s..e].iter());
734            let end = marked.len();
735            marked.push(close);
736            ranges.push(start..end);
737            cursor = e;
738        }
739        marked.extend(chars[cursor..].iter());
740        (marked, ranges)
741    }
742
743    proptest! {
744        // Marker collisions and multibyte-boundary splits are rare per sample, so hammer them with
745        // a high case count. The filtered strategies (distinct markers, marker-free clean text)
746        // reject frequently, so the reject caps and shrink budget are lifted well above default.
747        #![proptest_config(ProptestConfig {
748            cases: 2048,
749            max_shrink_iters: 8192,
750            max_local_rejects: 1 << 16,
751            max_global_rejects: 1 << 16,
752            ..ProptestConfig::default()
753        })]
754
755        // (A) never panics, and (E) every yielded range is an in-bounds, char-boundary, ascending,
756        // non-overlapping slice of the raw text — the load-bearing "ranges index the raw text"
757        // contract. `str::get(range)` returns Some only for an in-bounds, char-boundary,
758        // start <= end range, so it subsumes every bound except the non-overlap check.
759        #[test]
760        fn ranges_are_valid_ordered_slices_of_the_raw_text(
761            markers in distinct_markers(),
762            text in nasty_string(),
763        ) {
764            let h = TextHighlighter::with_markers(markers).unwrap();
765            let hl = h.as_highlighted(text.as_str());
766            let raw: &str = hl.as_ref();
767            let mut prev_end = 0usize;
768            for r in hl.ranges() {
769                prop_assert!(r.start >= prev_end);
770                prop_assert!(raw.get(r.clone()).is_some());
771                prev_end = r.end;
772            }
773        }
774
775        // (B) sanitize removes every marker, (C) is idempotent (a second pass borrows), and
776        // (D) equals the input with the marker chars filtered out.
777        #[test]
778        fn sanitize_is_a_marker_free_idempotent_filter(
779            markers in distinct_markers(),
780            text in nasty_string(),
781        ) {
782            let [open, close] = markers;
783            let h = TextHighlighter::with_markers(markers).unwrap();
784            let clean = h.sanitize(&text);
785            prop_assert!(!clean.contains([open, close]));
786            let filtered: String = text.chars().filter(|c| *c != open && *c != close).collect();
787            prop_assert_eq!(clean.as_ref(), filtered.as_str());
788            let again = h.sanitize(&clean);
789            prop_assert_eq!(again.as_ref(), clean.as_ref());
790            prop_assert!(matches!(again, Cow::Borrowed(_)));
791        }
792
793        // (F) round-trip: markers wrapped around chosen spans of marker-free text yield exactly
794        // those spans' byte ranges in the marked string, and sanitize strips them to recover the
795        // original text. Repeated span indices exercise empty spans (s == e) and adjacent spans.
796        #[test]
797        fn wrapping_spans_round_trips_through_ranges_and_sanitize(
798            (markers, chars, spans) in markers_clean_and_spans(),
799        ) {
800            let h = TextHighlighter::with_markers(markers).unwrap();
801            let (marked, expected) = wrap_spans(markers, &chars, &spans);
802            let clean: String = chars.iter().collect();
803            let stripped = h.sanitize(&marked);
804            prop_assert_eq!(stripped.as_ref(), clean.as_str());
805            let ranges = h.as_highlighted(marked.as_str()).ranges().collect::<Vec<_>>();
806            prop_assert_eq!(ranges, expected);
807        }
808
809        // (G) `pieces()` strips exactly the markers `display_plain` does, and its `Match` chunks are
810        // exactly the non-empty spans `ranges()` reports.
811        #[test]
812        fn pieces_reconstruct_plain_and_report_matches(
813            markers in distinct_markers(),
814            text in nasty_string(),
815        ) {
816            let h = TextHighlighter::with_markers(markers).unwrap();
817            let hl = h.as_highlighted(text.as_str());
818
819            let mut plain = String::new();
820            let mut matches: Vec<&str> = Vec::new();
821            for piece in hl.pieces() {
822                match piece {
823                    Piece::Text(t) => plain.push_str(t),
824                    Piece::Match(m) => {
825                        plain.push_str(m);
826                        matches.push(m);
827                    }
828                }
829            }
830            prop_assert_eq!(plain, hl.display_plain().to_string());
831
832            let raw: &str = hl.as_ref();
833            let want: Vec<&str> = hl.ranges().map(|r| &raw[r]).filter(|s| !s.is_empty()).collect();
834            prop_assert_eq!(matches, want);
835        }
836    }
837}
atuin_common/string/highlighted.rs

atuin_common/string/
highlighted.rs
