1use super::code_highlight_node::is_code_highlight_node;
10use super::code_indentation::register_code_indentation;
11use super::code_node::is_code_node;
12use super::code_tokenizer::{CodeLanguageState, Tokenizer};
13use super::flat_structure_utils::plainify_code_content;
14use crate::constants::NODE_STATE_KEY;
15use crate::error::invariant;
16use crate::lexical_selection::{PointType, Which, get_selection};
17use crate::node::NodeRef;
18use crate::node_state::get_writable_node_state;
19use crate::nodes::line_break::LineBreakNode;
20use crate::nodes::tab::TabNode;
21use crate::nodes::text::create_text_node;
22use crate::updates::on_update;
23use crate::{
24 Editor, HISTORY_MERGE_TAG, LexicalError, NodeKey, Unregister, UpdateOptions, get_node_by_key,
25};
26use std::cell::RefCell;
27use std::collections::HashSet;
28use std::rc::Rc;
29
30fn is_tab_node(node: &NodeRef) -> bool {
31 node.get_latest().is::<TabNode>()
32}
33
34fn is_line_break_node(node: &NodeRef) -> bool {
35 node.get_latest().is::<LineBreakNode>()
36}
37
38fn language_or_default(node: &NodeRef, tokenizer: &dyn Tokenizer) -> Option<String> {
40 node.get_language()
41 .or_else(|| tokenizer.default_language().map(str::to_owned))
42}
43
44fn copy_node_state(from: &NodeRef, to: &NodeRef) -> Result<(), LexicalError> {
48 let json = match &from.get_latest().base().state {
49 Some(state) => state.borrow().to_json(),
50 None => return Ok(()),
51 };
52 if json.is_empty() {
53 return Ok(());
54 }
55 let state = get_writable_node_state(to)?;
56 let mut state = state.borrow_mut();
57 for (key, value) in json.iter() {
58 match value.as_object() {
59 Some(nested) if key == NODE_STATE_KEY => {
60 for (nested_key, nested_value) in nested.iter() {
61 state.update_from_unknown(nested_key, Rc::new(nested_value.clone()));
62 }
63 }
64 _ => state.update_from_unknown(key, Rc::new(value.clone())),
65 }
66 }
67 Ok(())
68}
69
70#[derive(Default)]
74struct TransformState {
75 did_transform: bool,
76 nodes_currently_highlighting: HashSet<NodeKey>,
77 waiting_for_language: Vec<(String, Vec<NodeKey>)>,
81}
82
83struct Highlighter {
84 editor: std::rc::Weak<crate::editor::EditorInner>,
85 tokenizer: Rc<dyn Tokenizer>,
86 state: RefCell<TransformState>,
87}
88
89impl Highlighter {
90 fn text_node_transform(self: &Rc<Self>, node: &NodeRef) -> Result<(), LexicalError> {
92 let parent_node = node.get_parent();
94 if let Some(parent_node) = parent_node.filter(is_code_node) {
95 self.code_node_transform(&parent_node)
96 } else if is_code_highlight_node(node) {
97 let text = create_text_node(&node.text_data().text);
100 copy_node_state(node, &text)?;
101 node.replace(&text, false)?;
102 Ok(())
103 } else {
104 Ok(())
105 }
106 }
107
108 fn code_node_transform(self: &Rc<Self>, node: &NodeRef) -> Result<(), LexicalError> {
110 let tokenizer = &*self.tokenizer;
111 let node_key = node.get_key().to_owned();
112 if node.get_language().is_none()
116 && let Some(default_language) = tokenizer.default_language()
117 {
118 node.set_language(Some(default_language));
119 }
120 if let Some(language) = language_or_default(node, tokenizer) {
121 match tokenizer.language_state(&language) {
122 CodeLanguageState::Loaded => {
123 if !node.get_is_syntax_highlight_supported() {
124 node.set_is_syntax_highlight_supported(true);
125 }
126 }
127 CodeLanguageState::Loading => {
128 self.wait_for_language(language, node_key);
131 return Ok(());
132 }
133 CodeLanguageState::Unsupported => {
134 if node.get_is_syntax_highlight_supported() {
138 node.set_is_syntax_highlight_supported(false);
139 }
140 }
141 }
142 } else if node.get_is_syntax_highlight_supported() {
143 node.set_is_syntax_highlight_supported(false);
144 }
145
146 {
147 let mut state = self.state.borrow_mut();
148 if !state.nodes_currently_highlighting.insert(node_key.clone()) {
149 return Ok(());
150 }
151 if !state.did_transform {
152 state.did_transform = true;
153 let this = Rc::downgrade(self);
154 on_update(move || {
155 if let Some(this) = this.upgrade() {
156 let mut state = this.state.borrow_mut();
157 state.did_transform = false;
158 state.nodes_currently_highlighting.clear();
159 }
160 });
161 }
162 }
163
164 update_and_retain_selection(&node_key, || {
165 let Some(current_node) = get_node_by_key(&node_key) else {
166 return Ok(false);
167 };
168 if !is_code_node(¤t_node) || !current_node.is_attached() {
169 return Ok(false);
170 }
171 let current_language = language_or_default(¤t_node, tokenizer);
172 let highlight_nodes = match ¤t_language {
173 Some(language)
174 if tokenizer.language_state(language) != CodeLanguageState::Loaded =>
175 {
176 plainify_code_content(¤t_node.get_text_content())
177 }
178 _ => tokenizer.tokenize_code_node(¤t_node, current_language.as_deref()),
179 };
180 let (from, to, nodes_for_replacement) =
181 get_diff_range(¤t_node.get_children(), &highlight_nodes);
182 if from != to || !nodes_for_replacement.is_empty() {
183 node.splice(from, to - from, nodes_for_replacement)?;
184 return Ok(true);
185 }
186 Ok(false)
187 })
188 }
189
190 fn wait_for_language(self: &Rc<Self>, language: String, node_key: NodeKey) {
196 {
197 let mut state = self.state.borrow_mut();
198 if let Some((_, keys)) = state
199 .waiting_for_language
200 .iter_mut()
201 .find(|(l, _)| *l == language)
202 {
203 if !keys.contains(&node_key) {
204 keys.push(node_key);
205 }
206 return;
207 }
208 state
209 .waiting_for_language
210 .push((language.clone(), vec![node_key]));
211 }
212 let this = Rc::downgrade(self);
213 let loaded = language.clone();
214 self.tokenizer.load_language(
215 &language,
216 Box::new(move || {
217 let Some(this) = this.upgrade() else {
218 return;
219 };
220 let keys = {
221 let mut state = this.state.borrow_mut();
222 let index = state
223 .waiting_for_language
224 .iter()
225 .position(|(l, _)| *l == loaded);
226 index.map_or_else(Vec::new, |i| state.waiting_for_language.remove(i).1)
227 };
228 let Some(editor) = this.editor.upgrade().map(Editor) else {
229 return;
230 };
231 let tokenizer = this.tokenizer.clone();
232 let _ = editor.update(
233 move || {
234 for key in keys {
235 let Some(code_node) = get_node_by_key(&key).filter(is_code_node) else {
236 continue;
237 };
238 if language_or_default(&code_node, &*tokenizer).as_deref()
239 == Some(loaded.as_str())
240 {
241 code_node.set_is_syntax_highlight_supported(true);
242 code_node.mark_dirty();
243 }
244 }
245 Ok(())
246 },
247 UpdateOptions {
248 tags: vec![HISTORY_MERGE_TAG.to_owned()],
249 ..UpdateOptions::default()
250 },
251 );
252 }),
253 );
254 }
255}
256
257fn update_and_retain_selection(
261 node_key: &str,
262 update_fn: impl FnOnce() -> Result<bool, LexicalError>,
263) -> Result<(), LexicalError> {
264 let Some(node) = get_node_by_key(node_key) else {
265 return Ok(());
266 };
267 if !is_code_node(&node) || !node.is_attached() {
268 return Ok(());
269 }
270 let Some(selection) = get_selection().filter(|s| s.range().is_some()) else {
273 update_fn()?;
274 return Ok(());
275 };
276 let anchor = selection.point(Which::Anchor).expect("a range selection");
277 let anchor_node = anchor.get_node()?;
278 if anchor_node != node && !node.is_parent_of(&anchor_node) {
279 update_fn()?;
280 return Ok(());
281 }
282 let anchor_offset = anchor.offset;
283 let is_new_line_anchor = anchor.point_type == PointType::Element
284 && anchor_offset
285 .checked_sub(1)
286 .and_then(|i| node.get_child_at_index(i))
287 .is_some_and(|child| is_line_break_node(&child));
288 let mut text_offset = 0;
289 if !is_new_line_anchor {
292 text_offset = anchor_offset
293 + anchor_node
294 .get_previous_siblings()
295 .iter()
296 .map(NodeRef::get_text_content_size)
297 .sum::<usize>();
298 }
299
300 if !update_fn()? {
301 return Ok(());
302 }
303
304 if is_new_line_anchor {
306 let anchor = selection.point(Which::Anchor).expect("a range selection");
307 anchor
308 .get_node()?
309 .select(Some(anchor_offset), Some(anchor_offset))?;
310 return Ok(());
311 }
312 let children = node.get_children();
315 for (index, child) in children.iter().enumerate() {
316 if child.is_text() {
317 let text_content_size = child.get_text_content_size();
318 if text_content_size >= text_offset {
319 child.select(Some(text_offset), Some(text_offset))?;
320 return Ok(());
321 }
322 text_offset -= text_content_size;
323 } else if is_line_break_node(child) {
324 if text_offset == 0 {
325 node.select(Some(index), Some(index))?;
326 return Ok(());
327 }
328 text_offset -= 1;
329 }
330 }
331 node.select(Some(children.len()), Some(children.len()))?;
332 Ok(())
333}
334
335fn get_diff_range<'a>(
339 prev_nodes: &[NodeRef],
340 next_nodes: &'a [NodeRef],
341) -> (usize, usize, &'a [NodeRef]) {
342 let mut leading_match = 0;
343 while leading_match < prev_nodes.len() {
344 match next_nodes.get(leading_match) {
345 Some(next) if is_equal(&prev_nodes[leading_match], next) => leading_match += 1,
346 _ => break,
347 }
348 }
349 let prev_nodes_length = prev_nodes.len();
350 let next_nodes_length = next_nodes.len();
351 let max_trailing_match = prev_nodes_length.min(next_nodes_length) - leading_match;
352 let mut trailing_match = 0;
353 while trailing_match < max_trailing_match {
354 trailing_match += 1;
355 if !is_equal(
356 &prev_nodes[prev_nodes_length - trailing_match],
357 &next_nodes[next_nodes_length - trailing_match],
358 ) {
359 trailing_match -= 1;
360 break;
361 }
362 }
363 (
364 leading_match,
365 prev_nodes_length - trailing_match,
366 &next_nodes[leading_match..next_nodes_length - trailing_match],
367 )
368}
369
370fn is_equal(node_a: &NodeRef, node_b: &NodeRef) -> bool {
374 if is_code_highlight_node(node_a) && is_code_highlight_node(node_b) {
375 return node_a.get_text_content() == node_b.get_text_content()
376 && node_a.get_highlight_type() == node_b.get_highlight_type();
377 }
378 (is_tab_node(node_a) && is_tab_node(node_b))
379 || (is_line_break_node(node_a) && is_line_break_node(node_b))
380}
381
382pub fn register_highlighting_only(editor: &Editor, tokenizer: Rc<dyn Tokenizer>) -> Unregister {
388 let highlighter = Rc::new(Highlighter {
389 editor: editor.weak(),
390 tokenizer,
391 state: RefCell::default(),
392 });
393 let code = highlighter.clone();
394 let text = highlighter.clone();
395 let code_highlight = highlighter;
396 super::merge(vec![
397 editor.register_node_transform("code", move |node| code.code_node_transform(node)),
398 editor.register_node_transform("text", move |node| text.text_node_transform(node)),
399 editor.register_node_transform("code-highlight", move |node| {
400 code_highlight.text_node_transform(node)
401 }),
402 ])
403}
404
405pub fn register_code_highlighting(
412 editor: &Editor,
413 tokenizer: Rc<dyn Tokenizer>,
414) -> Result<Unregister, LexicalError> {
415 let nodes = editor.registered_nodes();
416 invariant!(
417 nodes.contains_key("code") && nodes.contains_key("code-highlight"),
418 "CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor"
419 );
420 Ok(super::merge(vec![
421 register_highlighting_only(editor, tokenizer),
422 register_code_indentation(editor, None, false),
423 ]))
424}
425
426#[cfg(feature = "extension")]
427pub use extension::*;
428
429#[cfg(feature = "extension")]
430mod extension {
431 use super::super::code_extension::{code_extension, code_indent_extension};
432 use super::super::code_node::DEFAULT_CODE_LANGUAGE;
433 use super::super::code_tokenizer::{PlainTokenizer, Tokenizer};
434 use super::register_highlighting_only;
435 use crate::Editor;
436 use crate::extension::named_signals_config;
437 use crate::extension::signals::effect;
438 use crate::extension_core::{
439 ExtensionBuildState, ExtensionRegisterState, LexicalExtension, define_extension,
440 };
441 use std::rc::Rc;
442
443 #[derive(Clone)]
446 pub struct TokenizerRef(pub Rc<dyn Tokenizer>);
447
448 impl PartialEq for TokenizerRef {
449 fn eq(&self, other: &Self) -> bool {
450 Rc::ptr_eq(&self.0, &other.0)
451 }
452 }
453
454 named_signals_config! {
455 pub struct CodeHighlighterConfig;
457 pub struct CodeHighlighterConfigOverrides;
459 pub struct CodeHighlighterOutput {
461 disabled: bool,
464 tokenizer: TokenizerRef,
466 }
467 }
468
469 pub type CodeHighlighterExtensionType =
471 LexicalExtension<CodeHighlighterConfig, CodeHighlighterOutput, ()>;
472
473 fn make_code_highlighter_extension() -> Rc<CodeHighlighterExtensionType> {
474 define_extension(LexicalExtension {
475 build: Some(Rc::new(
476 |_: &Editor, config: &CodeHighlighterConfig, _: &ExtensionBuildState<()>| {
477 Ok(CodeHighlighterOutput::of(config))
478 },
479 )),
480 dependencies: vec![code_extension().into(), code_indent_extension().into()],
481 register: Some(Rc::new(
482 |editor: &Editor,
483 _: &CodeHighlighterConfig,
484 state: &ExtensionRegisterState<(), CodeHighlighterOutput>| {
485 let stores = state
486 .get_output()
487 .expect("CodeHighlighterExtension has a build");
488 let editor = editor.weak();
489 Ok(effect(move || {
490 if stores.disabled.value() {
491 return None;
492 }
493 let tokenizer = stores.tokenizer.value();
494 let editor = editor.upgrade().map(Editor)?;
495 Some(register_highlighting_only(&editor, tokenizer.0))
496 }))
497 },
498 )),
499 ..LexicalExtension::new(
501 "@lexical/code-highlighter",
502 CodeHighlighterConfig {
503 disabled: false,
504 tokenizer: TokenizerRef(Rc::new(PlainTokenizer {
505 default_language: Some(DEFAULT_CODE_LANGUAGE.to_owned()),
506 })),
507 },
508 )
509 })
510 }
511
512 thread_local! {
513 static CODE_HIGHLIGHTER_EXTENSION: Rc<CodeHighlighterExtensionType> =
514 make_code_highlighter_extension();
515 }
516
517 pub fn code_highlighter_extension() -> Rc<CodeHighlighterExtensionType> {
524 CODE_HIGHLIGHTER_EXTENSION.with(Rc::clone)
525 }
526}
527
528#[cfg(test)]
529mod tests {
530 use super::super::code_node::create_code_node;
531 use super::super::code_nodes;
532 use super::super::code_tokenizer::{CodeToken, Tokenizer};
533 use super::*;
534 use crate::{CreateEditorArgs, create_editor, get_root};
535 use std::cell::Cell;
536
537 #[derive(Default)]
540 struct LazyTokenizer {
541 loaded: Cell<bool>,
542 loads: Cell<usize>,
543 on_ready: RefCell<Option<Box<dyn FnOnce()>>>,
544 }
545
546 impl Tokenizer for LazyTokenizer {
547 fn default_language(&self) -> Option<&str> {
548 Some("lazy")
549 }
550 fn language_state(&self, language: &str) -> CodeLanguageState {
551 match (language, self.loaded.get()) {
552 ("lazy", true) => CodeLanguageState::Loaded,
553 ("lazy", false) => CodeLanguageState::Loading,
554 _ => CodeLanguageState::Unsupported,
555 }
556 }
557 fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
558 vec![CodeToken::token("lazy", code)]
559 }
560 fn load_language(&self, _language: &str, on_ready: Box<dyn FnOnce()>) {
561 self.loads.set(self.loads.get() + 1);
562 *self.on_ready.borrow_mut() = Some(on_ready);
563 }
564 }
565
566 fn update(editor: &Editor, f: impl FnOnce() -> Result<(), LexicalError> + 'static) {
567 editor
568 .update(
569 f,
570 UpdateOptions {
571 discrete: true,
572 ..UpdateOptions::default()
573 },
574 )
575 .unwrap();
576 }
577
578 type Child = (String, String, Option<String>);
580
581 fn blocks(editor: &Editor) -> Vec<(bool, Vec<Child>)> {
583 editor
584 .read(|| {
585 get_root()
586 .get_children()
587 .iter()
588 .map(|block| {
589 let children = block
590 .get_children()
591 .iter()
592 .map(|c| {
593 (
594 c.get_type().to_owned(),
595 c.get_text_content(),
596 c.get_highlight_type(),
597 )
598 })
599 .collect();
600 (block.get_is_syntax_highlight_supported(), children)
601 })
602 .collect()
603 })
604 .unwrap()
605 }
606
607 #[test]
610 fn a_loading_language_highlights_once_loaded() {
611 let editor = create_editor(CreateEditorArgs {
612 nodes: code_nodes(),
613 ..CreateEditorArgs::default()
614 });
615 let tokenizer = Rc::new(LazyTokenizer::default());
616 let _unregister = register_code_highlighting(&editor, tokenizer.clone()).unwrap();
617 update(&editor, || {
618 for text in ["a", "b"] {
619 let code = create_code_node(None, None);
620 code.append(&[create_text_node(text)])?;
621 get_root().append(&[code])?;
622 }
623 Ok(())
624 });
625 let plain = |text: &str| vec![("text".to_owned(), text.to_owned(), None)];
626 assert_eq!(
627 blocks(&editor),
628 vec![(false, plain("a")), (false, plain("b"))]
629 );
630 assert_eq!(tokenizer.loads.get(), 1);
631 tokenizer.loaded.set(true);
632 let on_ready = tokenizer.on_ready.borrow_mut().take().unwrap();
633 on_ready();
634 editor.read(|| ()).unwrap();
635 let lazy = |text: &str| {
636 vec![(
637 "code-highlight".to_owned(),
638 text.to_owned(),
639 Some("lazy".to_owned()),
640 )]
641 };
642 assert_eq!(blocks(&editor), vec![(true, lazy("a")), (true, lazy("b"))]);
643 }
644
645 #[test]
647 fn registering_without_the_code_nodes_fails() {
648 let editor = create_editor(CreateEditorArgs::default());
649 let error = register_code_highlighting(&editor, Rc::new(LazyTokenizer::default()))
650 .err()
651 .unwrap();
652 assert_eq!(
653 error.message(),
654 "CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor"
655 );
656 }
657
658 #[cfg(feature = "extension")]
661 #[test]
662 fn the_extension_highlights_unless_disabled() {
663 use crate::extension::{
664 build_editor_from_extensions, get_extension_dependency_from_editor,
665 };
666 use crate::extension_core::config_extension;
667
668 struct Words;
669 impl Tokenizer for Words {
670 fn default_language(&self) -> Option<&str> {
671 Some("words")
672 }
673 fn language_state(&self, _language: &str) -> CodeLanguageState {
674 CodeLanguageState::Loaded
675 }
676 fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
677 vec![CodeToken::token("word", code)]
678 }
679 }
680 let built = build_editor_from_extensions(vec![config_extension(
681 &code_highlighter_extension(),
682 [CodeHighlighterConfigOverrides {
683 tokenizer: Some(TokenizerRef(Rc::new(Words))),
684 ..CodeHighlighterConfigOverrides::default()
685 }],
686 )])
687 .unwrap();
688 let editor = built.editor().clone();
689 editor.read(|| ()).unwrap();
690 update(&editor, || {
691 let code = create_code_node(None, None);
692 code.append(&[create_text_node("a")])?;
693 get_root().clear()?;
694 get_root().append(&[code])
695 });
696 let word = (
697 "code-highlight".to_owned(),
698 "a".to_owned(),
699 Some("word".to_owned()),
700 );
701 assert_eq!(blocks(&editor), vec![(true, vec![word.clone()])]);
702 let output = get_extension_dependency_from_editor(&editor, &code_highlighter_extension())
703 .unwrap()
704 .output
705 .unwrap();
706 output.disabled.set_value(true);
707 update(&editor, || {
708 let code = get_root().get_first_child().unwrap();
709 code.append(&[create_text_node("b")])
710 });
711 let text = ("text".to_owned(), "b".to_owned(), None);
712 assert_eq!(blocks(&editor), vec![(true, vec![word, text])]);
713 }
714}