Skip to main content

lexical_ui/code/
code_tokenizer.rs

1//! The tokenizer interface of `@lexical/code-prism` and
2//! `@lexical/code-shiki` (`Tokenizer`, with Prism's token shape), and
3//! `$mapTokensToLexicalStructure` (`FacadePrism.ts`), without a grammar
4//! engine: [`register_code_highlighting`](super::register_code_highlighting)
5//! runs the highlighting loop over any [`Tokenizer`] an app supplies, the
6//! way upstream lets Shiki stand in for Prism.
7//!
8//! Upstream's two engines differ in how a language becomes available. Prism
9//! has its grammars loaded or not (`isCodeLanguageLoaded`); Shiki downloads
10//! one on demand (`loadCodeLanguage`) and highlights the block again once it
11//! has loaded. [`CodeLanguageState`] covers both, as Swift's does.
12
13use super::code_highlight_node::create_code_highlight_node;
14use super::flat_structure_utils::plainify_code_content;
15use crate::node::NodeRef;
16use crate::nodes::line_break::create_line_break_node;
17use crate::nodes::tab::create_tab_node;
18use crate::selection_insert_nodes::{RawTextVisitor, tokenize_raw_text};
19
20/// `string | Token`: an item of what a tokenizer returns (Prism's
21/// `TokenStream`), text without a type or a typed [`Token`].
22#[derive(Clone, Debug, PartialEq)]
23pub enum CodeToken {
24    /// A string.
25    Text(String),
26    /// A `Token`.
27    Token(Token),
28}
29
30impl CodeToken {
31    /// A token of `token_type` whose content is the string `text`.
32    pub fn token(token_type: &str, text: &str) -> CodeToken {
33        CodeToken::Token(Token {
34            token_type: token_type.to_owned(),
35            alias: None,
36            content: TokenContent::Text(text.to_owned()),
37        })
38    }
39}
40
41/// `Token`: a token of `type` (a Prism token name such as `keyword`,
42/// `string`, `comment`, `function`, `number`, `operator` or
43/// `punctuation`).
44#[derive(Clone, Debug, PartialEq)]
45pub struct Token {
46    /// `type`.
47    pub token_type: String,
48    /// `alias` when it is a string: the type a `prefix` token with string
49    /// content gives its text (a diff's `inserted`, say). JS's alias can
50    /// also be an array, which the highlighter ignores, as it does `None`.
51    pub alias: Option<String>,
52    /// `content`.
53    pub content: TokenContent,
54}
55
56/// `TokenContent`: what a [`Token`] holds.
57#[derive(Clone, Debug, PartialEq)]
58pub enum TokenContent {
59    /// A string, typed by the token (by its alias for a `prefix`).
60    Text(String),
61    /// A token on its own, which `$mapTokensToLexicalStructure` leaves out,
62    /// as JS does.
63    Token(Box<Token>),
64    /// Tokens, typed by the token unless it is `unchanged` (a diff's
65    /// unchanged line, whose text has no type).
66    Tokens(Vec<CodeToken>),
67}
68
69/// Whether a tokenizer can highlight a language.
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum CodeLanguageState {
72    /// It has the language's grammar (`isCodeLanguageLoaded`).
73    Loaded,
74    /// It has the grammar, but not yet: [`Tokenizer::load_language`] gets
75    /// it, as Shiki's `loadCodeLanguage` imports a grammar.
76    Loading,
77    /// It doesn't know the language: the block is plain text.
78    Unsupported,
79}
80
81/// `Tokenizer`: what the highlighter asks a highlighting engine.
82pub trait Tokenizer {
83    /// `defaultLanguage`: the language of a code block without one, which
84    /// the highlighter sets on the block. `None` (JS's `null`) opts out:
85    /// such a block keeps no language and is split into plain tokens, so a
86    /// Markdown round trip keeps a fence without an info string.
87    fn default_language(&self) -> Option<&str>;
88
89    /// Whether the tokenizer can highlight `language` now. JS's loop asks
90    /// the engine's `isCodeLanguageLoaded`.
91    fn language_state(&self, language: &str) -> CodeLanguageState;
92
93    /// `tokenize(code, language)`: `code` as tokens, for a loaded
94    /// `language`.
95    fn tokenize(&self, code: &str, language: &str) -> Vec<CodeToken>;
96
97    /// Loads `language`, whose state is [`CodeLanguageState::Loading`], and
98    /// calls `on_ready` once it is loaded, on the editor's thread
99    /// (Shiki's `loadCodeLanguage`); the highlighter then highlights the
100    /// blocks waiting for it. The default does nothing, for a tokenizer
101    /// whose languages are all loaded or unsupported.
102    fn load_language(&self, language: &str, on_ready: Box<dyn FnOnce()>) {
103        let _ = (language, on_ready);
104    }
105
106    /// `$tokenize(codeNode, language)`: the children that highlight
107    /// `code_node`. The default is `PrismTokenizer.$tokenize`'s:
108    /// `language || defaultLanguage`, and with neither the plain tokens of
109    /// the block's text (`$plainifyCodeContent`), else
110    /// [`map_tokens_to_lexical_structure`] of
111    /// [`tokenize`](Self::tokenize) of it (`$getHighlightNodes`, without
112    /// Prism's `diff-<language>` grammars).
113    fn tokenize_code_node(&self, code_node: &NodeRef, language: Option<&str>) -> Vec<NodeRef> {
114        let code = code_node.get_text_content();
115        match language
116            .filter(|l| !l.is_empty())
117            .or(self.default_language())
118        {
119            None => plainify_code_content(&code),
120            Some(language) => {
121                map_tokens_to_lexical_structure(&self.tokenize(&code, language), None)
122            }
123        }
124    }
125}
126
127struct MapTokens<'a> {
128    nodes: &'a mut Vec<NodeRef>,
129    highlight_type: Option<&'a str>,
130}
131
132impl RawTextVisitor for MapTokens<'_> {
133    fn linebreak(&mut self) {
134        self.nodes.push(create_line_break_node());
135    }
136    fn tab(&mut self) {
137        self.nodes.push(create_tab_node());
138    }
139    fn text(&mut self, text: &str) {
140        self.nodes
141            .push(create_code_highlight_node(text, self.highlight_type));
142    }
143}
144
145/// `$mapTokensToLexicalStructure(tokens, type)` (`FacadePrism.ts`): the
146/// CodeHighlightNodes, TabNodes and LineBreakNodes of `tokens`, each string
147/// split at tabs and line breaks (`tokenizeRawText`) and typed by its
148/// innermost token: a `prefix` with string content by its alias, the
149/// tokens of an `unchanged` one by none.
150pub fn map_tokens_to_lexical_structure(
151    tokens: &[CodeToken],
152    highlight_type: Option<&str>,
153) -> Vec<NodeRef> {
154    let mut nodes = Vec::new();
155    add_tokens(tokens, highlight_type, &mut nodes);
156    nodes
157}
158
159fn add_tokens(tokens: &[CodeToken], highlight_type: Option<&str>, nodes: &mut Vec<NodeRef>) {
160    for token in tokens {
161        match token {
162            CodeToken::Text(text) => tokenize_raw_text(
163                text,
164                &mut MapTokens {
165                    nodes,
166                    highlight_type,
167                },
168            ),
169            CodeToken::Token(token) => match &token.content {
170                TokenContent::Text(text) => {
171                    let highlight_type = match (&token.alias, token.token_type.as_str()) {
172                        (Some(alias), "prefix") => alias.as_str(),
173                        _ => token.token_type.as_str(),
174                    };
175                    tokenize_raw_text(
176                        text,
177                        &mut MapTokens {
178                            nodes,
179                            highlight_type: Some(highlight_type),
180                        },
181                    );
182                }
183                TokenContent::Tokens(content) => add_tokens(
184                    content,
185                    (token.token_type != "unchanged").then_some(token.token_type.as_str()),
186                    nodes,
187                ),
188                TokenContent::Token(_) => {}
189            },
190        }
191    }
192}
193
194/// A tokenizer that knows no language: every code block is split into
195/// plain tokens, so the indent and line-move handlers can work on it. The
196/// default of `CodeHighlighterConfig`, as
197/// Swift's `PlainCodeTokenizer` is; JS's default is `PrismTokenizer`, whose
198/// grammars aren't ported.
199#[derive(Clone, Debug, Default)]
200pub struct PlainTokenizer {
201    /// `defaultLanguage`.
202    pub default_language: Option<String>,
203}
204
205impl Tokenizer for PlainTokenizer {
206    fn default_language(&self) -> Option<&str> {
207        self.default_language.as_deref()
208    }
209    fn language_state(&self, _language: &str) -> CodeLanguageState {
210        CodeLanguageState::Unsupported
211    }
212    fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
213        vec![CodeToken::Text(code.to_owned())]
214    }
215}