Skip to main content

gpui_base/input/base/
inline_tokens.rs

1//! Document annotations for atomic inline objects. Coordinates are always source bytes.
2use std::ops::Range;
3
4use gpui::{Context, EntityInputHandler as _, SharedString, Window};
5use unicode_segmentation::UnicodeSegmentation as _;
6
7use super::{InputBaseState, InputModeKind, undo_manager::EditIntent};
8
9/// An application-defined reference rendered as one inline editing unit. The
10/// ID names the referenced resource; the same ID may occur more than once.
11#[derive(Clone, Debug, PartialEq, Eq, Hash)]
12pub struct InlineToken {
13    id: SharedString,
14    text: SharedString,
15    label: SharedString,
16}
17
18impl InlineToken {
19    pub fn new(id: impl Into<SharedString>, text: impl Into<SharedString>) -> Self {
20        let text = text.into();
21        Self {
22            id: id.into(),
23            label: text.clone(),
24            text,
25        }
26    }
27    pub fn with_label(mut self, label: impl Into<SharedString>) -> Self {
28        self.label = label.into();
29        self
30    }
31    pub fn id(&self) -> &SharedString {
32        &self.id
33    }
34    pub fn text(&self) -> &SharedString {
35        &self.text
36    }
37    pub fn label(&self) -> &SharedString {
38        &self.label
39    }
40
41    fn validate(&self) -> Result<(), InlineTokenError> {
42        if self.id.trim().is_empty()
43            || [&self.text, &self.label].iter().any(|s| {
44                s.is_empty()
45                    || s.chars()
46                        .any(|c| c.is_control() || matches!(c, '\u{2028}' | '\u{2029}'))
47            })
48        {
49            return Err(InlineTokenError::InvalidToken);
50        }
51        Ok(())
52    }
53}
54
55/// A token and its current half-open UTF-8 byte range.
56#[derive(Clone, Debug, PartialEq, Eq)]
57pub struct InlineTokenSpan {
58    range: Range<usize>,
59    token: InlineToken,
60}
61impl InlineTokenSpan {
62    pub fn range(&self) -> Range<usize> {
63        self.range.clone()
64    }
65    pub fn token(&self) -> &InlineToken {
66        &self.token
67    }
68    fn shifted(&self, delta: isize) -> Self {
69        Self {
70            range: self
71                .range
72                .start
73                .checked_add_signed(delta)
74                .expect("valid token start")
75                ..self
76                    .range
77                    .end
78                    .checked_add_signed(delta)
79                    .expect("valid token end"),
80            token: self.token.clone(),
81        }
82    }
83}
84
85/// An owned, coherent text-and-token snapshot: what `content()` returns and
86/// what `set_value` accepts. Plain text converts into content without tokens,
87/// and every token is validated against the text as it is attached, so a
88/// content value is always consistent.
89#[derive(Clone, Debug, PartialEq, Eq)]
90pub struct InputContent {
91    text: SharedString,
92    tokens: Vec<InlineTokenSpan>,
93}
94impl InputContent {
95    pub fn new(text: impl Into<SharedString>) -> Self {
96        Self {
97            text: text.into(),
98            tokens: vec![],
99        }
100    }
101    /// Attach a token to a half-open UTF-8 byte range of the text. The range
102    /// must be nonempty, sit on grapheme boundaries, contain exactly the
103    /// token's text and not overlap another token.
104    pub fn with_token(
105        mut self,
106        range: Range<usize>,
107        token: InlineToken,
108    ) -> Result<Self, InlineTokenError> {
109        token.validate()?;
110        validate_range(&self.text, &range)?;
111        if range.is_empty() {
112            return Err(InlineTokenError::InvalidRange);
113        }
114        if &self.text[range.clone()] != token.text.as_ref() {
115            return Err(InlineTokenError::TextMismatch);
116        }
117        let ix = self
118            .tokens
119            .partition_point(|span| span.range.end <= range.start);
120        if self
121            .tokens
122            .get(ix)
123            .is_some_and(|span| span.range.start < range.end)
124        {
125            return Err(InlineTokenError::OverlappingTokens);
126        }
127        self.tokens.insert(ix, InlineTokenSpan { range, token });
128        Ok(self)
129    }
130    pub fn text(&self) -> &SharedString {
131        &self.text
132    }
133    pub fn tokens(&self) -> &[InlineTokenSpan] {
134        &self.tokens
135    }
136}
137
138macro_rules! content_from_text {
139    ($($text:ty),* $(,)?) => {
140        $(impl From<$text> for InputContent {
141            fn from(text: $text) -> Self {
142                Self::new(text)
143            }
144        })*
145    };
146}
147// Every text type `set_value` accepted before it took content.
148content_from_text!(
149    &str,
150    &mut str,
151    &String,
152    String,
153    Box<str>,
154    std::sync::Arc<str>,
155    &std::sync::Arc<str>,
156    std::borrow::Cow<'_, str>,
157    &SharedString,
158    SharedString,
159);
160
161/// A rejected token operation never partially changes the document.
162#[derive(Clone, Copy, Debug, PartialEq, Eq)]
163#[non_exhaustive]
164pub enum InlineTokenError {
165    InvalidRange,
166    InvalidBoundary,
167    InvalidToken,
168    OverlappingTokens,
169    TextMismatch,
170    UnsupportedMode,
171    ValidationRejected,
172    CompositionActive,
173}
174
175impl std::error::Error for InlineTokenError {}
176impl std::fmt::Display for InlineTokenError {
177    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
178        f.write_str(match self {
179            Self::InvalidRange => "token range is outside the document or empty",
180            Self::InvalidBoundary => "token range splits a Unicode grapheme",
181            Self::InvalidToken => {
182                "token requires a nonempty ID, text and label without control characters"
183            }
184            Self::OverlappingTokens => "token ranges overlap",
185            Self::TextMismatch => "token text does not match its range or input normalization",
186            Self::UnsupportedMode => "tokens are not supported by this input mode",
187            Self::ValidationRejected => "input validation rejected the content",
188            Self::CompositionActive => "finish the active IME composition before editing tokens",
189        })
190    }
191}
192
193fn validate_range(text: &str, range: &Range<usize>) -> Result<(), InlineTokenError> {
194    if range.start > range.end || range.end > text.len() {
195        return Err(InlineTokenError::InvalidRange);
196    }
197    let boundary =
198        |offset| offset == text.len() || text.grapheme_indices(true).any(|(ix, _)| ix == offset);
199    if !boundary(range.start) || !boundary(range.end) {
200        return Err(InlineTokenError::InvalidBoundary);
201    }
202    Ok(())
203}
204
205#[derive(Default)]
206pub(super) struct InlineTokenStore {
207    spans: Vec<InlineTokenSpan>,
208}
209
210/// Only affected records are retained in history. Ranges are relative to the edit start.
211#[derive(Clone, Debug, PartialEq, Eq)]
212pub(super) struct TokenDelta {
213    removed: Vec<InlineTokenSpan>,
214    inserted: Vec<InlineTokenSpan>,
215}
216
217impl InlineTokenStore {
218    fn replace(
219        &mut self,
220        range: &Range<usize>,
221        new_len: usize,
222        inserted: &[InlineTokenSpan],
223    ) -> Option<Box<TokenDelta>> {
224        let mut removed = Vec::new();
225        let shift = new_len as isize - range.len() as isize;
226        self.spans.retain_mut(|span| {
227            if span.range.start < range.end && range.start < span.range.end {
228                removed.push(span.shifted(-(range.start as isize)));
229                false
230            } else {
231                if span.range.start >= range.end {
232                    *span = span.shifted(shift);
233                }
234                true
235            }
236        });
237        self.spans
238            .extend(inserted.iter().map(|s| s.shifted(range.start as isize)));
239        if !inserted.is_empty() {
240            self.spans.sort_by_key(|s| s.range.start);
241        }
242        (!removed.is_empty() || !inserted.is_empty()).then(|| {
243            Box::new(TokenDelta {
244                removed,
245                inserted: inserted.to_vec(),
246            })
247        })
248    }
249}
250
251impl<M: InputModeKind> InputBaseState<M> {
252    pub(super) fn token_spans(&self) -> &[InlineTokenSpan] {
253        self.inline_tokens
254            .as_ref()
255            .map_or(&[], |store| &store.spans)
256    }
257    pub(super) fn token_boundary(&self, offset: usize, bias: sum_tree::Bias) -> usize {
258        let spans = self.token_spans();
259        let ix = spans.partition_point(|s| s.range.end <= offset);
260        if let Some(span) = spans.get(ix).filter(|s| s.range.start < offset) {
261            if bias == sum_tree::Bias::Left {
262                span.range.start
263            } else {
264                span.range.end
265            }
266        } else {
267            offset
268        }
269    }
270    pub(super) fn normalize_token_range(&self, range: Range<usize>) -> Range<usize> {
271        if self.replaying_history {
272            return range;
273        }
274        if range.is_empty() {
275            let offset = self.token_boundary(range.start, sum_tree::Bias::Right);
276            offset..offset
277        } else {
278            self.token_boundary(range.start, sum_tree::Bias::Left)
279                ..self.token_boundary(range.end, sum_tree::Bias::Right)
280        }
281    }
282    pub(super) fn edit_tokens(
283        &mut self,
284        range: &Range<usize>,
285        new_len: usize,
286    ) -> Option<Box<TokenDelta>> {
287        if self.replaying_history {
288            return None;
289        }
290        let inserted = self.pending_token.take().map(|token| InlineTokenSpan {
291            range: 0..new_len,
292            token,
293        });
294        if inserted.is_some() && self.inline_tokens.is_none() {
295            self.inline_tokens = Some(Box::default());
296        }
297        self.inline_tokens
298            .as_mut()?
299            .replace(range, new_len, inserted.as_slice())
300    }
301    pub(super) fn replay_tokens(
302        &mut self,
303        range: &Range<usize>,
304        new_len: usize,
305        delta: Option<&TokenDelta>,
306        undo: bool,
307    ) {
308        let inserted = delta
309            .map(|d| {
310                if undo {
311                    &d.removed[..]
312                } else {
313                    &d.inserted[..]
314                }
315            })
316            .unwrap_or_default();
317        if !inserted.is_empty() && self.inline_tokens.is_none() {
318            self.inline_tokens = Some(Box::default());
319        }
320        if let Some(store) = self.inline_tokens.as_mut() {
321            store.replace(range, new_len, inserted);
322        }
323    }
324    fn check_token_mode(&self) -> Result<(), InlineTokenError> {
325        if self.ime_marked_range.is_some() {
326            return Err(InlineTokenError::CompositionActive);
327        }
328        if M::CODE_EDITOR || self.masked || self.token_is_secret() || !self.mask_pattern.is_none() {
329            return Err(InlineTokenError::UnsupportedMode);
330        }
331        Ok(())
332    }
333    fn replace_token(
334        &mut self,
335        range: Range<usize>,
336        token: InlineToken,
337        window: &mut Window,
338        cx: &mut Context<Self>,
339    ) -> Result<(), InlineTokenError> {
340        self.check_token_mode()?;
341        token.validate()?;
342        let text = self.text.to_string();
343        validate_range(&text, &range)?;
344        let range = self.normalize_token_range(range);
345        let mut next = text;
346        next.replace_range(range.clone(), &token.text);
347        validate_range(&next, &(range.start..range.start + token.text.len()))?;
348        if self.normalize_input(&next) != next {
349            return Err(InlineTokenError::TextMismatch);
350        }
351        if !self.is_valid_input(&next, cx) {
352            return Err(InlineTokenError::ValidationRejected);
353        }
354        if self
355            .token_spans()
356            .iter()
357            .any(|s| s.range == range && s.token == token)
358        {
359            return Ok(());
360        }
361        let new_text = token.text.clone();
362        self.pending_token = Some(token);
363        self.undo_manager.break_transaction_coalescing();
364        self.undo_manager.set_pending_intent(EditIntent::Atomic);
365        let range_utf16 = self.range_to_utf16(&range);
366        self.validated_token_edit = true;
367        self.with_edits_allowed(|state| {
368            state.replace_text_in_range(Some(range_utf16), &new_text, window, cx)
369        });
370        self.validated_token_edit = false;
371        self.pending_token = None;
372        Ok(())
373    }
374    /// Adopt the tokens of content whose text was just installed by
375    /// `set_value`. Tokens are dropped when this mode cannot show them or when
376    /// normalization changed the text, since their ranges would no longer
377    /// describe it.
378    pub(super) fn install_tokens(&mut self, content: InputContent) {
379        let supported = !M::CODE_EDITOR && self.mask_pattern.is_none();
380        let text_kept = self.text == content.text.as_ref();
381        self.inline_tokens = (supported && text_kept && !content.tokens.is_empty()).then(|| {
382            Box::new(InlineTokenStore {
383                spans: content.tokens,
384            })
385        });
386    }
387}
388
389macro_rules! token_api {
390    ($mode:ty) => {
391        impl InputBaseState<$mode> {
392            /// Replace the selection with an atomic token; no delimiter is added.
393            pub fn replace_with_token(
394                &mut self,
395                token: InlineToken,
396                window: &mut Window,
397                cx: &mut Context<Self>,
398            ) -> Result<(), InlineTokenError> {
399                self.replace_token(self.selected_range(), token, window, cx)
400            }
401            /// Replace a UTF-8 byte range, expanding overlaps to whole tokens.
402            pub fn replace_range_with_token(
403                &mut self,
404                range: Range<usize>,
405                token: InlineToken,
406                window: &mut Window,
407                cx: &mut Context<Self>,
408            ) -> Result<(), InlineTokenError> {
409                self.replace_token(range, token, window, cx)
410            }
411            pub fn tokens(&self) -> &[InlineTokenSpan] {
412                self.token_spans()
413            }
414            /// The text with its tokens, as `set_value` accepts it.
415            pub fn content(&self) -> InputContent {
416                InputContent {
417                    text: self.value(),
418                    tokens: self.token_spans().to_vec(),
419                }
420            }
421        }
422    };
423}
424token_api!(super::InputMode);
425token_api!(super::TextareaMode);