lexical_ui/code/code_tokenizer.rs
1//! The tokenizer interface of `@lexical/code-prism` and
2//! `@lexical/code-shiki` (`Tokenizer`, with Prism's token shape), and
3//! `$mapTokensToLexicalStructure` (`FacadePrism.ts`), without a grammar
4//! engine: [`register_code_highlighting`](super::register_code_highlighting)
5//! runs the highlighting loop over any [`Tokenizer`] an app supplies, the
6//! way upstream lets Shiki stand in for Prism.
7//!
8//! Upstream's two engines differ in how a language becomes available. Prism
9//! has its grammars loaded or not (`isCodeLanguageLoaded`); Shiki downloads
10//! one on demand (`loadCodeLanguage`) and highlights the block again once it
11//! has loaded. [`CodeLanguageState`] covers both, as Swift's does.
12
13use super::code_highlight_node::create_code_highlight_node;
14use super::flat_structure_utils::plainify_code_content;
15use crate::node::NodeRef;
16use crate::nodes::line_break::create_line_break_node;
17use crate::nodes::tab::create_tab_node;
18use crate::selection_insert_nodes::{RawTextVisitor, tokenize_raw_text};
19
20/// `string | Token`: an item of what a tokenizer returns (Prism's
21/// `TokenStream`), text without a type or a typed [`Token`].
22#[derive(Clone, Debug, PartialEq)]
23pub enum CodeToken {
24 /// A string.
25 Text(String),
26 /// A `Token`.
27 Token(Token),
28}
29
30impl CodeToken {
31 /// A token of `token_type` whose content is the string `text`.
32 pub fn token(token_type: &str, text: &str) -> CodeToken {
33 CodeToken::Token(Token {
34 token_type: token_type.to_owned(),
35 alias: None,
36 content: TokenContent::Text(text.to_owned()),
37 })
38 }
39}
40
41/// `Token`: a token of `type` (a Prism token name such as `keyword`,
42/// `string`, `comment`, `function`, `number`, `operator` or
43/// `punctuation`).
44#[derive(Clone, Debug, PartialEq)]
45pub struct Token {
46 /// `type`.
47 pub token_type: String,
48 /// `alias` when it is a string: the type a `prefix` token with string
49 /// content gives its text (a diff's `inserted`, say). JS's alias can
50 /// also be an array, which the highlighter ignores, as it does `None`.
51 pub alias: Option<String>,
52 /// `content`.
53 pub content: TokenContent,
54}
55
56/// `TokenContent`: what a [`Token`] holds.
57#[derive(Clone, Debug, PartialEq)]
58pub enum TokenContent {
59 /// A string, typed by the token (by its alias for a `prefix`).
60 Text(String),
61 /// A token on its own, which `$mapTokensToLexicalStructure` leaves out,
62 /// as JS does.
63 Token(Box<Token>),
64 /// Tokens, typed by the token unless it is `unchanged` (a diff's
65 /// unchanged line, whose text has no type).
66 Tokens(Vec<CodeToken>),
67}
68
69/// Whether a tokenizer can highlight a language.
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum CodeLanguageState {
72 /// It has the language's grammar (`isCodeLanguageLoaded`).
73 Loaded,
74 /// It has the grammar, but not yet: [`Tokenizer::load_language`] gets
75 /// it, as Shiki's `loadCodeLanguage` imports a grammar.
76 Loading,
77 /// It doesn't know the language: the block is plain text.
78 Unsupported,
79}
80
81/// `Tokenizer`: what the highlighter asks a highlighting engine.
82pub trait Tokenizer {
83 /// `defaultLanguage`: the language of a code block without one, which
84 /// the highlighter sets on the block. `None` (JS's `null`) opts out:
85 /// such a block keeps no language and is split into plain tokens, so a
86 /// Markdown round trip keeps a fence without an info string.
87 fn default_language(&self) -> Option<&str>;
88
89 /// Whether the tokenizer can highlight `language` now. JS's loop asks
90 /// the engine's `isCodeLanguageLoaded`.
91 fn language_state(&self, language: &str) -> CodeLanguageState;
92
93 /// `tokenize(code, language)`: `code` as tokens, for a loaded
94 /// `language`.
95 fn tokenize(&self, code: &str, language: &str) -> Vec<CodeToken>;
96
97 /// Loads `language`, whose state is [`CodeLanguageState::Loading`], and
98 /// calls `on_ready` once it is loaded, on the editor's thread
99 /// (Shiki's `loadCodeLanguage`); the highlighter then highlights the
100 /// blocks waiting for it. The default does nothing, for a tokenizer
101 /// whose languages are all loaded or unsupported.
102 fn load_language(&self, language: &str, on_ready: Box<dyn FnOnce()>) {
103 let _ = (language, on_ready);
104 }
105
106 /// `$tokenize(codeNode, language)`: the children that highlight
107 /// `code_node`. The default is `PrismTokenizer.$tokenize`'s:
108 /// `language || defaultLanguage`, and with neither the plain tokens of
109 /// the block's text (`$plainifyCodeContent`), else
110 /// [`map_tokens_to_lexical_structure`] of
111 /// [`tokenize`](Self::tokenize) of it (`$getHighlightNodes`, without
112 /// Prism's `diff-<language>` grammars).
113 fn tokenize_code_node(&self, code_node: &NodeRef, language: Option<&str>) -> Vec<NodeRef> {
114 let code = code_node.get_text_content();
115 match language
116 .filter(|l| !l.is_empty())
117 .or(self.default_language())
118 {
119 None => plainify_code_content(&code),
120 Some(language) => {
121 map_tokens_to_lexical_structure(&self.tokenize(&code, language), None)
122 }
123 }
124 }
125}
126
127struct MapTokens<'a> {
128 nodes: &'a mut Vec<NodeRef>,
129 highlight_type: Option<&'a str>,
130}
131
132impl RawTextVisitor for MapTokens<'_> {
133 fn linebreak(&mut self) {
134 self.nodes.push(create_line_break_node());
135 }
136 fn tab(&mut self) {
137 self.nodes.push(create_tab_node());
138 }
139 fn text(&mut self, text: &str) {
140 self.nodes
141 .push(create_code_highlight_node(text, self.highlight_type));
142 }
143}
144
145/// `$mapTokensToLexicalStructure(tokens, type)` (`FacadePrism.ts`): the
146/// CodeHighlightNodes, TabNodes and LineBreakNodes of `tokens`, each string
147/// split at tabs and line breaks (`tokenizeRawText`) and typed by its
148/// innermost token: a `prefix` with string content by its alias, the
149/// tokens of an `unchanged` one by none.
150pub fn map_tokens_to_lexical_structure(
151 tokens: &[CodeToken],
152 highlight_type: Option<&str>,
153) -> Vec<NodeRef> {
154 let mut nodes = Vec::new();
155 add_tokens(tokens, highlight_type, &mut nodes);
156 nodes
157}
158
159fn add_tokens(tokens: &[CodeToken], highlight_type: Option<&str>, nodes: &mut Vec<NodeRef>) {
160 for token in tokens {
161 match token {
162 CodeToken::Text(text) => tokenize_raw_text(
163 text,
164 &mut MapTokens {
165 nodes,
166 highlight_type,
167 },
168 ),
169 CodeToken::Token(token) => match &token.content {
170 TokenContent::Text(text) => {
171 let highlight_type = match (&token.alias, token.token_type.as_str()) {
172 (Some(alias), "prefix") => alias.as_str(),
173 _ => token.token_type.as_str(),
174 };
175 tokenize_raw_text(
176 text,
177 &mut MapTokens {
178 nodes,
179 highlight_type: Some(highlight_type),
180 },
181 );
182 }
183 TokenContent::Tokens(content) => add_tokens(
184 content,
185 (token.token_type != "unchanged").then_some(token.token_type.as_str()),
186 nodes,
187 ),
188 TokenContent::Token(_) => {}
189 },
190 }
191 }
192}
193
194/// A tokenizer that knows no language: every code block is split into
195/// plain tokens, so the indent and line-move handlers can work on it. The
196/// default of `CodeHighlighterConfig`, as
197/// Swift's `PlainCodeTokenizer` is; JS's default is `PrismTokenizer`, whose
198/// grammars aren't ported.
199#[derive(Clone, Debug, Default)]
200pub struct PlainTokenizer {
201 /// `defaultLanguage`.
202 pub default_language: Option<String>,
203}
204
205impl Tokenizer for PlainTokenizer {
206 fn default_language(&self) -> Option<&str> {
207 self.default_language.as_deref()
208 }
209 fn language_state(&self, _language: &str) -> CodeLanguageState {
210 CodeLanguageState::Unsupported
211 }
212 fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
213 vec![CodeToken::Text(code.to_owned())]
214 }
215}