Skip to main content

oxc_parser/
markup.rs

1//! Host-embedded markup: markup written inside a JS/TS program.
2//!
3//! This is the reverse of embedding script in markup. The host grammar stays
4//! the owner of every expression position. When it reaches a position where a
5//! registered [`MarkupGrammar`] opens a region, it hands that position to the
6//! grammar, which parses the region and returns its end. The host keeps an
7//! empty placeholder whose span is exactly the region. While parsing, the
8//! grammar asks this same parser, through a [`MarkupCursor`], to parse every
9//! embedded expression, statement, binding and head, so those nodes are
10//! ordinary AST nodes in the same arena with original spans.
11//!
12//! Positions that can open a region:
13//! - a `<` where JSX is enabled ([`MarkupSite::Expression`]);
14//! - an `@` at the start of an expression or statement, when the grammar opens
15//!   it ([`MarkupSite::Expression`]);
16//! - an `@` where a function body would start ([`MarkupSite::FunctionBody`]).
17//!
18//! While a grammar is registered, a `<` that starts a line and is followed by a
19//! tag-like character ends the preceding expression (so automatic semicolon
20//! insertion separates `const a = b` from a following `<tag>`), matching the
21//! line-start rule of markup-in-script languages.
22
23use oxc_allocator::{Allocator, ArenaBox, ArenaVec, GetAllocator};
24use oxc_ast::ast::*;
25use oxc_diagnostics::OxcDiagnostic;
26use oxc_span::Span;
27
28use crate::{
29    Context, Parser, ParserConfig as Config, ParserImpl, ParserReturn, StatementContext,
30    config::NoTokensParserConfig,
31    diagnostics::{self, ParserDiagnostic},
32    js::{VariableDeclarationParent, grammar::CoverGrammar},
33    lexer::{Kind, Token},
34};
35
36/// Where a markup region opens in the host grammar.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum MarkupSite {
39    /// An expression (or expression statement) position.
40    Expression,
41    /// A function, method or arrow body position.
42    FunctionBody,
43}
44
45/// A markup grammar embedded in host JS/TS.
46///
47/// The grammar is shared (`&self`) so a region may re-enter the host parser,
48/// which may open nested regions with the same grammar.
49pub trait MarkupGrammar<'a> {
50    /// Whether a region opens at `offset` (the start of an `@` token) for `site`.
51    /// A `<` in a JSX-enabled profile always opens an expression region.
52    fn opens(&self, source: &'a str, offset: u32, site: MarkupSite) -> bool;
53
54    /// Parse one region that starts at `cursor.token()`, and return the offset where
55    /// it ends. Errors are reported with [`MarkupCursor::fatal`].
56    fn region(&self, cursor: &mut MarkupCursor<'_, 'a>, site: MarkupSite) -> u32;
57}
58
59/// The host operations a markup grammar may use inside one region.
60pub struct MarkupCursor<'p, 'a> {
61    host: &'p mut dyn MarkupHost<'a>,
62}
63
64trait MarkupHost<'a> {
65    fn source(&self) -> &'a str;
66    fn allocator(&self) -> &'a Allocator;
67    fn token(&self) -> Token;
68    fn seat(&mut self, offset: u32) -> bool;
69    fn bump(&mut self);
70    fn expression(&mut self) -> Expression<'a>;
71    fn sequence(&mut self) -> Expression<'a>;
72    fn statement(&mut self) -> Statement<'a>;
73    fn for_of_head(&mut self) -> Option<(ForStatementLeft<'a>, Expression<'a>)>;
74    fn binding_identifier(&mut self) -> BindingIdentifier<'a>;
75    fn formal_parameters(&mut self) -> ArenaBox<'a, FormalParameters<'a>>;
76    fn type_arguments(&mut self) -> Option<ArenaBox<'a, TSTypeParameterInstantiation<'a>>>;
77    fn fatal(&mut self, diagnostic: OxcDiagnostic);
78    fn failed(&self) -> bool;
79}
80
81impl<'a> MarkupCursor<'_, 'a> {
82    /// The whole host source.
83    pub fn source(&self) -> &'a str {
84        self.host.source()
85    }
86
87    /// The arena of the host AST.
88    pub fn allocator(&self) -> &'a Allocator {
89        self.host.allocator()
90    }
91
92    /// The current host token.
93    pub fn token(&self) -> Token {
94        self.host.token()
95    }
96
97    /// Move the host to `offset` and read the token there (after trivia).
98    /// Returns `false` (and reports a fatal error) when `offset` is not a
99    /// character boundary of the source.
100    pub fn seat(&mut self, offset: u32) -> bool {
101        self.host.seat(offset)
102    }
103
104    /// Advance past the current host token.
105    pub fn bump(&mut self) {
106        self.host.bump();
107    }
108
109    /// Parse an `AssignmentExpression` at the current token, with `in` allowed.
110    pub fn expression(&mut self) -> Expression<'a> {
111        self.host.expression()
112    }
113
114    /// Parse an `Expression` (a comma sequence) at the current token.
115    pub fn sequence(&mut self) -> Expression<'a> {
116        self.host.sequence()
117    }
118
119    /// Parse one `StatementListItem` at the current token.
120    pub fn statement(&mut self) -> Statement<'a> {
121        self.host.statement()
122    }
123
124    /// Parse `( ForDeclaration of AssignmentExpression` at the current `(`, leaving
125    /// the token after the iterable (a `;` clause or the closing `)`).
126    /// Returns `None`, leaving the token after the left side, when the head is not a for-of head.
127    pub fn for_of_head(&mut self) -> Option<(ForStatementLeft<'a>, Expression<'a>)> {
128        self.host.for_of_head()
129    }
130
131    /// Parse a `BindingIdentifier` at the current token.
132    pub fn binding_identifier(&mut self) -> BindingIdentifier<'a> {
133        self.host.binding_identifier()
134    }
135
136    /// Parse parenthesized `FormalParameters` at the current `(`.
137    pub fn formal_parameters(&mut self) -> ArenaBox<'a, FormalParameters<'a>> {
138        self.host.formal_parameters()
139    }
140
141    /// Parse `<Type, …>` type arguments at the current `<` (a TSRX tag's
142    /// `<List<T>>`). Returns `None`, leaving the token, when it is not a `<`.
143    pub fn type_arguments(&mut self) -> Option<ArenaBox<'a, TSTypeParameterInstantiation<'a>>> {
144        self.host.type_arguments()
145    }
146
147    /// Report an error that ends the parse.
148    pub fn fatal(&mut self, diagnostic: OxcDiagnostic) {
149        self.host.fatal(diagnostic);
150    }
151
152    /// Whether the parse has already failed fatally.
153    pub fn failed(&self) -> bool {
154        self.host.failed()
155    }
156}
157
158impl<'a, C: Config> MarkupHost<'a> for ParserImpl<'a, C> {
159    fn source(&self) -> &'a str {
160        self.source_text
161    }
162
163    fn allocator(&self) -> &'a Allocator {
164        GetAllocator::allocator(self)
165    }
166
167    fn token(&self) -> Token {
168        self.token
169    }
170
171    fn seat(&mut self, offset: u32) -> bool {
172        self.seat_at(offset)
173    }
174
175    fn bump(&mut self) {
176        self.bump_any();
177    }
178
179    fn expression(&mut self) -> Expression<'a> {
180        self.context_add(Context::In, ParserImpl::parse_assignment_expression_or_higher)
181    }
182
183    fn sequence(&mut self) -> Expression<'a> {
184        self.context_add(Context::In, ParserImpl::parse_expr)
185    }
186
187    fn statement(&mut self) -> Statement<'a> {
188        self.parse_statement_list_item(StatementContext::StatementList)
189    }
190
191    fn for_of_head(&mut self) -> Option<(ForStatementLeft<'a>, Expression<'a>)> {
192        self.parse_markup_for_of_head()
193    }
194
195    fn binding_identifier(&mut self) -> BindingIdentifier<'a> {
196        self.parse_binding_identifier()
197    }
198
199    fn formal_parameters(&mut self) -> ArenaBox<'a, FormalParameters<'a>> {
200        let (_, parameters) = self.parse_formal_parameters(
201            crate::js::FunctionKind::Expression,
202            FormalParameterKind::FormalParameter,
203        );
204        parameters
205    }
206
207    fn type_arguments(&mut self) -> Option<ArenaBox<'a, TSTypeParameterInstantiation<'a>>> {
208        self.try_parse_type_arguments()
209    }
210
211    fn fatal(&mut self, diagnostic: OxcDiagnostic) {
212        self.set_fatal_error(ParserDiagnostic::Eager(Box::new(diagnostic)));
213    }
214
215    fn failed(&self) -> bool {
216        self.fatal_error.is_some()
217    }
218}
219
220impl<'a, C: Config> ParserImpl<'a, C> {
221    /// Move to `offset` and make the token there current.
222    pub(crate) fn seat_at(&mut self, offset: u32) -> bool {
223        if let Some(token) = self.lexer.seat(offset) {
224            self.prev_token_end = offset;
225            self.token = token;
226            true
227        } else {
228            self.set_fatal_error(diagnostics::unexpected_token(Span::empty(offset)));
229            false
230        }
231    }
232
233    /// Whether the current `@` opens a markup region for `site`.
234    #[inline]
235    pub(crate) fn at_markup_directive(&self, site: MarkupSite) -> bool {
236        C::MARKUP && self.at(Kind::At) && self.markup_opens(site)
237    }
238
239    #[cold]
240    #[inline(never)]
241    fn markup_opens(&self, site: MarkupSite) -> bool {
242        self.markup
243            .is_some_and(|grammar| grammar.opens(self.source_text, self.cur_token().start(), site))
244    }
245
246    /// Whether the current `<` starts a line and a tag while a grammar is registered.
247    #[inline]
248    pub(crate) fn at_markup_line_start_tag(&self) -> bool {
249        C::MARKUP && self.markup_line_start_tag()
250    }
251
252    #[cold]
253    #[inline(never)]
254    fn markup_line_start_tag(&self) -> bool {
255        let token = self.cur_token();
256        token.kind() == Kind::LAngle
257            && token.is_on_new_line()
258            && self.source_text.as_bytes().get(token.end() as usize).is_some_and(|byte| {
259                byte.is_ascii_alphabetic() || matches!(byte, b'_' | b'$' | b'>' | b'{')
260            })
261    }
262
263    /// Run the registered grammar on the region at the current token; return its end.
264    fn run_markup(&mut self, site: MarkupSite) -> u32 {
265        let start = self.cur_token().start();
266        let Some(grammar) = self.markup else {
267            self.set_unexpected();
268            return start;
269        };
270        let end = grammar.region(&mut MarkupCursor { host: self }, site);
271        if self.fatal_error.is_none() && (end <= start || !self.seat_at(end)) {
272            self.set_fatal_error(diagnostics::unexpected_token(Span::empty(start)));
273        }
274        end.max(start)
275    }
276
277    /// An expression-position region, kept as an empty fragment with the region's span.
278    #[cold]
279    #[inline(never)]
280    pub(crate) fn parse_markup_expression(&mut self) -> Expression<'a> {
281        let start = self.cur_token().start();
282        let end = self.run_markup(MarkupSite::Expression);
283        let opening = JSXOpeningFragment::new(Span::empty(start), self);
284        let closing = JSXClosingFragment::new(Span::empty(end), self);
285        Expression::JSXFragment(JSXFragment::boxed(
286            Span::new(start, end),
287            opening,
288            ArenaVec::new_in(self),
289            closing,
290            self,
291        ))
292    }
293
294    /// A function-body region, kept as an empty body with the region's span.
295    #[cold]
296    #[inline(never)]
297    pub(crate) fn parse_markup_function_body(&mut self) -> ArenaBox<'a, FunctionBody<'a>> {
298        let start = self.cur_token().start();
299        let end = self.context(Context::Return, Context::TopLevel, |p| {
300            p.run_markup(MarkupSite::FunctionBody)
301        });
302        FunctionBody::boxed(
303            Span::new(start, end),
304            ArenaVec::new_in(self),
305            ArenaVec::new_in(self),
306            self,
307        )
308    }
309
310    fn parse_markup_for_of_head(&mut self) -> Option<(ForStatementLeft<'a>, Expression<'a>)> {
311        self.expect(Kind::LParen);
312        let declaration_kind = match self.cur_kind() {
313            Kind::Const => Some(VariableDeclarationKind::Const),
314            Kind::Let => Some(VariableDeclarationKind::Let),
315            Kind::Var => Some(VariableDeclarationKind::Var),
316            _ => None,
317        };
318        let left = if let Some(kind) = declaration_kind {
319            let start = self.start_span();
320            self.bump_any();
321            let declaration = self.context_remove(Context::In, |p| {
322                p.parse_variable_declaration(start, kind, VariableDeclarationParent::For, false)
323            });
324            ForStatementLeft::VariableDeclaration(declaration)
325        } else {
326            let target =
327                self.context_remove(Context::In, ParserImpl::parse_lhs_expression_or_higher);
328            ForStatementLeft::from(AssignmentTarget::cover(target, self))
329        };
330        if !self.at(Kind::Of) {
331            return None;
332        }
333        self.bump_any();
334        let right =
335            self.context_add(Context::In, ParserImpl::parse_assignment_expression_or_higher);
336        Some((left, right))
337    }
338}
339
340impl<'a> Parser<'a, NoTokensParserConfig> {
341    /// Parse a program whose markup regions belong to `grammar`.
342    ///
343    /// The result is an ordinary [`ParserReturn`]. It never mints
344    /// [`crate::ProgramObservation`] admission: its program contains
345    /// placeholders that only the grammar's own artifact can interpret.
346    pub fn parse_with_markup(self, grammar: &'a dyn MarkupGrammar<'a>) -> ParserReturn<'a> {
347        crate::parse_with_markup(
348            self.allocator,
349            self.source_text,
350            self.source_type,
351            self.options,
352            grammar,
353        )
354    }
355}
356
357#[cfg(test)]
358mod tests {
359    use std::cell::RefCell;
360
361    use oxc_allocator::Allocator;
362    use oxc_ast::ast::{Expression, Statement};
363    use oxc_span::{GetSpan, SourceType};
364
365    use super::{MarkupCursor, MarkupGrammar, MarkupSite};
366    use crate::{Kind, Parser};
367
368    type Recorded = Vec<(MarkupSite, u32, u32, String)>;
369
370    /// `<{ expression }>` regions and `@{ statements }` bodies.
371    #[derive(Default)]
372    struct Toy {
373        regions: RefCell<Recorded>,
374    }
375
376    impl<'a> MarkupGrammar<'a> for Toy {
377        fn opens(&self, source: &'a str, offset: u32, _: MarkupSite) -> bool {
378            source.as_bytes().get(offset as usize + 1) == Some(&b'{')
379        }
380
381        fn region(&self, cursor: &mut MarkupCursor<'_, 'a>, site: MarkupSite) -> u32 {
382            let start = cursor.token().start();
383            cursor.seat(start + 2);
384            let (inner, end) = if cursor.source().as_bytes()[start as usize] == b'<' {
385                let expression = cursor.expression();
386                let close = cursor.token();
387                assert_eq!(close.kind(), Kind::RCurly);
388                (expression.span().source_text(cursor.source()).to_string(), close.end() + 1)
389            } else {
390                let mut statements = Vec::new();
391                while cursor.token().kind() != Kind::RCurly {
392                    let statement = cursor.statement();
393                    statements.push(statement.span().source_text(cursor.source()).to_string());
394                }
395                (statements.join("|"), cursor.token().end())
396            };
397            self.regions.borrow_mut().push((site, start, end, inner));
398            end
399        }
400    }
401
402    fn parse(source: &str) -> (Recorded, usize, Option<(u32, u32)>) {
403        let toy = Toy::default();
404        let allocator = Allocator::default();
405        let parsed = Parser::new(&allocator, source, SourceType::tsx()).parse_with_markup(&toy);
406        assert!(parsed.diagnostics.is_empty(), "{source}: {:?}", parsed.diagnostics);
407        let placeholder = match parsed.program.body.first() {
408            Some(Statement::VariableDeclaration(declaration)) => {
409                match declaration.declarations[0].init.as_ref() {
410                    Some(Expression::JSXFragment(fragment)) => {
411                        assert!(fragment.children.is_empty());
412                        Some((fragment.span.start, fragment.span.end))
413                    }
414                    _ => None,
415                }
416            }
417            Some(Statement::FunctionDeclaration(function)) => {
418                let body = function.body.as_ref().unwrap();
419                assert!(body.statements.is_empty());
420                Some((body.span.start, body.span.end))
421            }
422            _ => None,
423        };
424        let regions = toy.regions.borrow().clone();
425        (regions, parsed.program.body.len(), placeholder)
426    }
427
428    #[test]
429    fn regions_keep_exact_placeholders_and_reenter_the_host() {
430        let (regions, statements, placeholder) = parse("const a = <{b + [<{c}>][0]}>;");
431        assert_eq!(statements, 1);
432        assert_eq!(placeholder, Some((10, 28)));
433        assert_eq!(
434            regions,
435            [
436                (MarkupSite::Expression, 17, 22, "c".to_string()),
437                (MarkupSite::Expression, 10, 28, "b + [<{c}>][0]".to_string()),
438            ]
439        );
440    }
441
442    #[test]
443    fn function_bodies_and_statement_starts_open_regions() {
444        let (regions, _, placeholder) = parse("function f() @{ const x = 1; g(x) }");
445        assert_eq!(placeholder, Some((13, 35)));
446        assert_eq!(regions, [(MarkupSite::FunctionBody, 13, 35, "const x = 1;|g(x)".to_string())]);
447        let (regions, statements, _) = parse("let v;\n@{ v = 1 }\n@dec class K {}");
448        assert_eq!(statements, 3);
449        assert_eq!(regions, [(MarkupSite::Expression, 7, 17, "v = 1".to_string())]);
450    }
451
452    #[test]
453    fn a_line_start_tag_ends_the_previous_expression() {
454        let (regions, statements, _) = parse("const a = b\n<{c}>\nconst d = e < f;");
455        assert_eq!(statements, 3);
456        assert_eq!(regions, [(MarkupSite::Expression, 12, 17, "c".to_string())]);
457    }
458}