m2_syn/parser/
realtime.rs

1//! Retained native parsing over independently cached source cells.
2//!
3//! [`NativeRealtimeParser`] treats a confirmed cell boundary as a restart
4//! point. Edits invalidate the cell containing the edit and only the suffix
5//! whose framing changes. The final EOF boundary remains speculative, so an
6//! append reopens that cell while leaving every confirmed predecessor intact.
7
8use std::error::Error;
9use std::fmt::{Display, Formatter};
10use std::mem;
11use std::ops::Range;
12use std::sync::Arc;
13
14use crate::cell_stream::{end_span, ends_with_cell_semicolon, frame_recovering};
15use crate::native::{
16    NativeCellState, NativeParseError, incomplete_lex_error, incomplete_parse_error, parse_raw_cell,
17};
18use crate::{
19    Cell, CellStream, CellTokenStream, LexError, Source, Span, Spanned, TextEdit,
20    lex_str_recovering,
21};
22
23/// The structural boundary following one retained native cell.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
25pub enum NativeCellBoundary {
26    /// A newline or semicolon confirmed the end of this cell.
27    EOC,
28    /// The current end of input provisionally ends otherwise complete syntax.
29    EOF,
30    /// The cell still requires more input and therefore has no closing boundary.
31    Unbounded,
32}
33
34/// A source edit that does not fit the current realtime buffer.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36pub struct NativeRealtimeEditError;
37
38impl Display for NativeRealtimeEditError {
39    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
40        formatter.write_str("source edit is outside the current realtime buffer")
41    }
42}
43
44impl Error for NativeRealtimeEditError {}
45
46/// One independently retained cell in a [`NativeRealtimeParser`].
47///
48/// Spans inside the token and typed representations are local to the exact
49/// source returned by [`Self::source`]. [`Self::document_range`] locates that
50/// immutable cell source in the current document. This separation permits an
51/// unchanged cell to survive edits above it without rewriting every leaf span.
52#[derive(Debug)]
53pub struct NativeRealtimeCell {
54    /// The document range.
55    document_range: Range<usize>,
56    /// The source.
57    source: Span,
58    /// The tokens.
59    tokens: CellTokenStream,
60    /// The syntax.
61    syntax: Option<Cell>,
62    /// The lexer diagnostics.
63    lexer_diagnostics: Vec<LexError>,
64    /// The parse error.
65    parse_error: Option<NativeParseError>,
66    /// The state.
67    state: NativeCellState,
68    /// The boundary.
69    boundary: NativeCellBoundary,
70}
71
72impl NativeRealtimeCell {
73    /// Returns this cell's exact source spelling.
74    ///
75    /// # Panics
76    ///
77    /// Panics if the internally retained source span cannot resolve its text.
78    pub fn source(&self) -> &str {
79        self.source
80            .text()
81            .expect("realtime cell source spans must resolve to text")
82    }
83
84    /// Returns this cell's half-open byte range in the current document.
85    pub fn document_range(&self) -> Range<usize> {
86        self.document_range.clone()
87    }
88
89    /// Returns the tokenized cell, including inferred adjacency punctuation.
90    pub fn tokens(&self) -> &CellTokenStream {
91        &self.tokens
92    }
93
94    /// Returns recovered typed syntax when the native parser could construct it.
95    pub fn syntax(&self) -> Option<&Cell> {
96        self.syntax.as_ref()
97    }
98
99    /// Returns recoverable lexer diagnostics associated with this cell.
100    pub fn lexer_diagnostics(&self) -> &[LexError] {
101        &self.lexer_diagnostics
102    }
103
104    /// Returns the native parser failure associated with this cell.
105    pub fn parse_error(&self) -> Option<&NativeParseError> {
106        self.parse_error.as_ref()
107    }
108
109    /// Returns whether this cell is complete, incomplete, or invalid.
110    pub fn state(&self) -> NativeCellState {
111        self.state
112    }
113
114    /// Returns the structural boundary currently following this cell.
115    pub fn boundary(&self) -> NativeCellBoundary {
116        self.boundary
117    }
118}
119
120impl Spanned for NativeRealtimeCell {
121    fn try_span(&self) -> Option<Span> {
122        Some(self.source.clone())
123    }
124}
125
126/// A pseudo-incremental native parser retaining independently framed cells.
127///
128/// Appending to an EOF-complete cell reparses only that speculative cell.
129/// Confirmed cells above it retain their token and typed allocations. An edit
130/// inside an earlier cell reparses strictly forwards; after cell framing
131/// stabilizes, an unchanged suffix is reused in its entirety. Adjacency and
132/// cell boundaries are regenerated together for every reparsed region.
133#[derive(Debug)]
134pub struct NativeRealtimeParser {
135    /// The complete current source snapshot.
136    source: String,
137    /// Independently framed cells covering the current source.
138    cells: Vec<NativeRealtimeCell>,
139    /// The cell range rebuilt by the most recent operation.
140    reparsed_cells: Range<usize>,
141}
142
143impl Default for NativeRealtimeParser {
144    fn default() -> Self {
145        Self::new("")
146    }
147}
148
149impl NativeRealtimeParser {
150    /// Parses and retains an initial source snapshot.
151    pub fn new(source: impl Into<String>) -> Self {
152        let source = source.into();
153        let cells = analyze_region(&source, 0);
154        let reparsed_cells = 0..cells.len();
155        Self {
156            source,
157            cells,
158            reparsed_cells,
159        }
160    }
161
162    /// Applies an edit and reparses only the forward cell region it invalidates.
163    ///
164    /// The operation is transactional when the edit is outside the source or
165    /// crosses a UTF-8 character boundary. Invalid Macaulay2 syntax is retained
166    /// as [`NativeCellState::Invalid`] rather than failing the edit operation.
167    ///
168    /// # Errors
169    ///
170    /// Returns [`NativeRealtimeEditError`] when `edit` cannot be applied to the
171    /// current source buffer.
172    pub fn apply_edit(
173        &mut self,
174        edit: &TextEdit,
175    ) -> Result<NativeCellState, NativeRealtimeEditError> {
176        let source = edit.apply(&self.source).ok_or(NativeRealtimeEditError)?;
177        let first_changed = self.first_affected_cell(edit);
178        let region_start = self
179            .cells
180            .get(first_changed)
181            .map_or(self.source.len(), |cell| cell.document_range.start);
182        let reusable_suffix = self.reusable_suffix(edit, &source, first_changed, region_start);
183        let rebuild_end = reusable_suffix
184            .map(|(_, start)| start)
185            .unwrap_or(source.len());
186        let rebuilt = analyze_region(&source[region_start..rebuild_end], region_start);
187
188        let mut previous = mem::take(&mut self.cells);
189        let previous_suffix = previous.split_off(first_changed);
190        let rebuilt_len = rebuilt.len();
191        previous.extend(rebuilt);
192        let reuse_from = reusable_suffix
193            .map(|(index, _)| index.saturating_sub(first_changed))
194            .unwrap_or(previous_suffix.len());
195        for mut retained in previous_suffix.into_iter().skip(reuse_from) {
196            retained.document_range = shifted_range(edit, retained.document_range);
197            previous.push(retained);
198        }
199
200        self.source = source;
201        self.cells = previous;
202        self.reparsed_cells = first_changed..first_changed + rebuilt_len;
203        Ok(self.state())
204    }
205
206    /// Appends source text at the current EOF.
207    ///
208    /// # Panics
209    ///
210    /// Panics if the parser's retained source length is not a valid insertion
211    /// boundary, which would violate its internal UTF-8 source invariant.
212    pub fn append(&mut self, source: impl Into<String>) -> NativeCellState {
213        let edit = TextEdit::insertion(self.source.len(), source);
214        self.apply_edit(&edit)
215            .expect("an insertion at the current EOF is always valid")
216    }
217
218    /// Returns the complete current source buffer.
219    pub fn source(&self) -> &str {
220        &self.source
221    }
222
223    /// Returns every retained complete, incomplete, or invalid cell.
224    pub fn cells(&self) -> &[NativeRealtimeCell] {
225        &self.cells
226    }
227
228    /// Returns the range of newly analyzed cells after the most recent edit.
229    ///
230    /// Cells outside this range were retained from the preceding snapshot.
231    pub fn reparsed_cells(&self) -> Range<usize> {
232        self.reparsed_cells.clone()
233    }
234
235    /// Returns the aggregate state of the current source snapshot.
236    pub fn state(&self) -> NativeCellState {
237        if self
238            .cells
239            .iter()
240            .any(|cell| cell.state == NativeCellState::Invalid)
241        {
242            NativeCellState::Invalid
243        } else if self
244            .cells
245            .last()
246            .is_some_and(|cell| cell.state == NativeCellState::Incomplete)
247        {
248            NativeCellState::Incomplete
249        } else if self.cells.is_empty() {
250            NativeCellState::Empty
251        } else {
252            NativeCellState::Complete
253        }
254    }
255
256    /// Projects the retained cells as the parser-facing tokenized stream.
257    ///
258    /// The projection includes an incomplete or invalid tail so a parser or
259    /// macro can inspect the exact current tokenization.
260    pub fn cell_stream(&self) -> CellStream {
261        CellStream::from_cells(self.cells.iter().map(|cell| cell.tokens.clone()))
262    }
263
264    /// Handles first affected cell.
265    fn first_affected_cell(&self, edit: &TextEdit) -> usize {
266        let start = match edit {
267            TextEdit::Insertion { at, .. } => *at,
268            TextEdit::Deletion { start, .. } => *start,
269        };
270        self.cells
271            .iter()
272            .position(|cell| {
273                start < cell.document_range.end
274                    || (start == cell.document_range.end
275                        && cell.boundary != NativeCellBoundary::EOC)
276            })
277            .unwrap_or(self.cells.len())
278    }
279
280    /// Handles reusable suffix.
281    fn reusable_suffix(
282        &self,
283        edit: &TextEdit,
284        source: &str,
285        first_changed: usize,
286        region_start: usize,
287    ) -> Option<(usize, usize)> {
288        for index in first_changed.saturating_add(1)..self.cells.len() {
289            let old_start = self.cells[index].document_range.start;
290            let new_start = shifted_offset(edit, old_start);
291            if new_start < region_start
292                || self.source.get(old_start..) != source.get(new_start..)
293                || !ends_at_confirmed_boundary(&source[region_start..new_start])
294            {
295                continue;
296            }
297            return Some((index, new_start));
298        }
299        None
300    }
301}
302
303/// Handles shifted range.
304fn shifted_range(edit: &TextEdit, range: Range<usize>) -> Range<usize> {
305    shifted_offset(edit, range.start)..shifted_offset(edit, range.end)
306}
307
308/// Handles shifted offset.
309fn shifted_offset(edit: &TextEdit, offset: usize) -> usize {
310    match edit {
311        TextEdit::Insertion { at, text } if offset >= *at => offset + text.len(),
312        TextEdit::Insertion { .. } => offset,
313        TextEdit::Deletion { start, .. } if offset <= *start => offset,
314        TextEdit::Deletion { start, end } if offset <= *end => *start,
315        TextEdit::Deletion { start, end } => offset - (end - start),
316    }
317}
318
319/// Returns whether the value ends at confirmed boundary.
320fn ends_at_confirmed_boundary(source: &str) -> bool {
321    if source.trim().is_empty() {
322        return true;
323    }
324    let Ok(recovered) = lex_str_recovering(source) else {
325        return false;
326    };
327    if !recovered.diagnostics.is_empty() {
328        return false;
329    }
330    let framed = frame_recovering(recovered.value);
331    if framed.incomplete.is_some() {
332        return false;
333    }
334    let Some(last) = framed.cells.last() else {
335        return true;
336    };
337    ends_with_cell_semicolon(last) || source.ends_with('\n')
338}
339
340/// Handles analyze region.
341fn analyze_region(source: &str, document_start: usize) -> Vec<NativeRealtimeCell> {
342    let recovered = match lex_str_recovering(source) {
343        Ok(recovered) => recovered,
344        Err(error) => {
345            if source.is_empty() {
346                return Vec::new();
347            }
348            return vec![cell_from_lex_error(
349                source,
350                document_start..document_start + source.len(),
351                error,
352            )];
353        }
354    };
355    let lexically_incomplete = !recovered.diagnostics.is_empty();
356    let framed = frame_recovering(recovered.value);
357    let complete_count = framed.cells.len();
358    let has_incomplete = framed.incomplete.is_some();
359    let mut start = 0;
360    let mut cells = Vec::with_capacity(complete_count + usize::from(has_incomplete));
361
362    for (index, tokens) in framed.cells.into_iter().enumerate() {
363        let has_following = index + 1 < complete_count || has_incomplete;
364        let mut end = end_span(&tokens)
365            .end()
366            .unwrap_or(source.len())
367            .min(source.len());
368        if !has_following && end < source.len() {
369            end = source.len();
370        }
371        let boundary = if !has_following && lexically_incomplete {
372            NativeCellBoundary::Unbounded
373        } else if has_following
374            || ends_with_cell_semicolon(&tokens)
375            || source[start..end].ends_with('\n')
376        {
377            NativeCellBoundary::EOC
378        } else {
379            NativeCellBoundary::EOF
380        };
381        cells.push(analyze_cell(
382            &source[start..end],
383            document_start + start..document_start + end,
384            boundary,
385        ));
386        start = end;
387    }
388
389    if framed.incomplete.is_some() {
390        cells.push(analyze_cell(
391            &source[start..],
392            document_start + start..document_start + source.len(),
393            NativeCellBoundary::Unbounded,
394        ));
395    }
396    cells
397}
398/// Handles analyze cell
399fn analyze_cell(
400    source: &str,
401    document_range: Range<usize>,
402    boundary: NativeCellBoundary,
403) -> NativeRealtimeCell {
404    let recovered = match lex_str_recovering(source) {
405        Ok(recovered) => recovered,
406        Err(error) => return cell_from_lex_error(source, document_range, error),
407    };
408    let diagnostics = recovered.diagnostics;
409    let framed = frame_recovering(recovered.value);
410    let mut streams = framed.cells;
411    streams.extend(framed.incomplete);
412    let tokens = match streams.len() {
413        0 => CellTokenStream::default(),
414        1 => streams.pop().expect("one framed stream is present"),
415        _ => CellTokenStream::from_framed_cells(streams),
416    };
417    let source_owner = tokens.source().unwrap_or_else(|| Source::new(source));
418    let source_span = Span::ranged(source_owner, 0..source.len());
419
420    let (syntax, parse_error, state) = match parse_raw_cell(tokens.clone()) {
421        Ok(syntax) => {
422            let state = if boundary == NativeCellBoundary::Unbounded || !diagnostics.is_empty() {
423                NativeCellState::Incomplete
424            } else {
425                NativeCellState::Complete
426            };
427            (syntax, None, state)
428        }
429        Err(error) => {
430            let state = if boundary != NativeCellBoundary::EOC && incomplete_parse_error(&error) {
431                NativeCellState::Incomplete
432            } else {
433                NativeCellState::Invalid
434            };
435            (None, Some(error), state)
436        }
437    };
438
439    NativeRealtimeCell {
440        document_range,
441        source: source_span,
442        tokens,
443        syntax,
444        lexer_diagnostics: diagnostics,
445        parse_error,
446        state,
447        boundary,
448    }
449}
450
451/// Handles cell from lex error.
452fn cell_from_lex_error(
453    source: &str,
454    document_range: Range<usize>,
455    error: LexError,
456) -> NativeRealtimeCell {
457    let state = if incomplete_lex_error(error.kind()) {
458        NativeCellState::Incomplete
459    } else {
460        NativeCellState::Invalid
461    };
462    let source_owner = Arc::clone(error.span().source());
463    NativeRealtimeCell {
464        document_range,
465        source: Span::ranged(source_owner, 0..source.len()),
466        tokens: CellTokenStream::default(),
467        syntax: None,
468        lexer_diagnostics: vec![error],
469        parse_error: None,
470        state,
471        boundary: NativeCellBoundary::Unbounded,
472    }
473}
m2_syn/parser/realtime.rs

m2_syn/parser/
realtime.rs
