Skip to main content

lexical_ui/code/
code_highlighter.rs

1//! The highlighting loop of
2//! `packages/lexical-code-prism/src/CodeHighlighterPrism.ts`, with the lazy
3//! languages of `packages/lexical-code-shiki/src/CodeHighlighterShiki.ts`,
4//! over any [`Tokenizer`] rather than Prism: a code block's text is
5//! tokenized again when it changes, only the children that differ from the
6//! new tokens are replaced, and a caret inside the block keeps its text
7//! offset.
8
9use super::code_highlight_node::is_code_highlight_node;
10use super::code_indentation::register_code_indentation;
11use super::code_node::is_code_node;
12use super::code_tokenizer::{CodeLanguageState, Tokenizer};
13use super::flat_structure_utils::plainify_code_content;
14use crate::constants::NODE_STATE_KEY;
15use crate::error::invariant;
16use crate::lexical_selection::{PointType, Which, get_selection};
17use crate::node::NodeRef;
18use crate::node_state::get_writable_node_state;
19use crate::nodes::line_break::LineBreakNode;
20use crate::nodes::tab::TabNode;
21use crate::nodes::text::create_text_node;
22use crate::updates::on_update;
23use crate::{
24    Editor, HISTORY_MERGE_TAG, LexicalError, NodeKey, Unregister, UpdateOptions, get_node_by_key,
25};
26use std::cell::RefCell;
27use std::collections::HashSet;
28use std::rc::Rc;
29
30fn is_tab_node(node: &NodeRef) -> bool {
31    node.get_latest().is::<TabNode>()
32}
33
34fn is_line_break_node(node: &NodeRef) -> bool {
35    node.get_latest().is::<LineBreakNode>()
36}
37
38/// `language || defaultLanguage`: a code block's language never is `""`.
39fn language_or_default(node: &NodeRef, tokenizer: &dyn Tokenizer) -> Option<String> {
40    node.get_language()
41        .or_else(|| tokenizer.default_language().map(str::to_owned))
42}
43
44/// `$copyNodeState(from, to)`: gives `to` the NodeState of `from`, flat
45/// states included, keeping the state `to` has for the keys `from` doesn't
46/// set.
47fn copy_node_state(from: &NodeRef, to: &NodeRef) -> Result<(), LexicalError> {
48    let json = match &from.get_latest().base().state {
49        Some(state) => state.borrow().to_json(),
50        None => return Ok(()),
51    };
52    if json.is_empty() {
53        return Ok(());
54    }
55    let state = get_writable_node_state(to)?;
56    let mut state = state.borrow_mut();
57    for (key, value) in json.iter() {
58        match value.as_object() {
59            Some(nested) if key == NODE_STATE_KEY => {
60                for (nested_key, nested_value) in nested.iter() {
61                    state.update_from_unknown(nested_key, Rc::new(nested_value.clone()));
62                }
63            }
64            _ => state.update_from_unknown(key, Rc::new(value.clone())),
65        }
66    }
67    Ok(())
68}
69
70/// `TransformState`. `nodesCurrentlyHighlighting` keeps the CodeNode and
71/// text transforms, which both run when a token is inserted, from
72/// highlighting a block twice in one update.
73#[derive(Default)]
74struct TransformState {
75    did_transform: bool,
76    nodes_currently_highlighting: HashSet<NodeKey>,
77    /// The blocks waiting for each language that is loading, so that a
78    /// language is loaded once however many transforms ask for it (Swift's;
79    /// Shiki's `loadCodeLanguage` loads per block).
80    waiting_for_language: Vec<(String, Vec<NodeKey>)>,
81}
82
83struct Highlighter {
84    editor: std::rc::Weak<crate::editor::EditorInner>,
85    tokenizer: Rc<dyn Tokenizer>,
86    state: RefCell<TransformState>,
87}
88
89impl Highlighter {
90    /// `$textNodeTransform(editor, tokenizer, transformState, node)`.
91    fn text_node_transform(self: &Rc<Self>, node: &NodeRef) -> Result<(), LexicalError> {
92        // A CodeNode's children are flat, so only the parent needs checking.
93        let parent_node = node.get_parent();
94        if let Some(parent_node) = parent_node.filter(is_code_node) {
95            self.code_node_transform(&parent_node)
96        } else if is_code_highlight_node(node) {
97            // A code block turned into another block: its tokens are text
98            // again, keeping their NodeState (such as comment ids).
99            let text = create_text_node(&node.text_data().text);
100            copy_node_state(node, &text)?;
101            node.replace(&text, false)?;
102            Ok(())
103        } else {
104            Ok(())
105        }
106    }
107
108    /// `$codeNodeTransform(editor, tokenizer, transformState, node)`.
109    fn code_node_transform(self: &Rc<Self>, node: &NodeRef) -> Result<(), LexicalError> {
110        let tokenizer = &*self.tokenizer;
111        let node_key = node.get_key().to_owned();
112        // A new block may have no language. A tokenizer without a default
113        // leaves it unset, so Markdown round-trips a fence without an info
114        // string.
115        if node.get_language().is_none()
116            && let Some(default_language) = tokenizer.default_language()
117        {
118            node.set_language(Some(default_language));
119        }
120        if let Some(language) = language_or_default(node, tokenizer) {
121            match tokenizer.language_state(&language) {
122                CodeLanguageState::Loaded => {
123                    if !node.get_is_syntax_highlight_supported() {
124                        node.set_is_syntax_highlight_supported(true);
125                    }
126                }
127                CodeLanguageState::Loading => {
128                    // The grammar is on its way (Shiki's `loadCodeLanguage`):
129                    // the block is highlighted once it has loaded.
130                    self.wait_for_language(language, node_key);
131                    return Ok(());
132                }
133                CodeLanguageState::Unsupported => {
134                    // An unknown language ("```After ") falls through to
135                    // plain tokens, so the indent and line-move handlers can
136                    // work on it.
137                    if node.get_is_syntax_highlight_supported() {
138                        node.set_is_syntax_highlight_supported(false);
139                    }
140                }
141            }
142        } else if node.get_is_syntax_highlight_supported() {
143            node.set_is_syntax_highlight_supported(false);
144        }
145
146        {
147            let mut state = self.state.borrow_mut();
148            if !state.nodes_currently_highlighting.insert(node_key.clone()) {
149                return Ok(());
150            }
151            if !state.did_transform {
152                state.did_transform = true;
153                let this = Rc::downgrade(self);
154                on_update(move || {
155                    if let Some(this) = this.upgrade() {
156                        let mut state = this.state.borrow_mut();
157                        state.did_transform = false;
158                        state.nodes_currently_highlighting.clear();
159                    }
160                });
161            }
162        }
163
164        update_and_retain_selection(&node_key, || {
165            let Some(current_node) = get_node_by_key(&node_key) else {
166                return Ok(false);
167            };
168            if !is_code_node(&current_node) || !current_node.is_attached() {
169                return Ok(false);
170            }
171            let current_language = language_or_default(&current_node, tokenizer);
172            let highlight_nodes = match &current_language {
173                Some(language)
174                    if tokenizer.language_state(language) != CodeLanguageState::Loaded =>
175                {
176                    plainify_code_content(&current_node.get_text_content())
177                }
178                _ => tokenizer.tokenize_code_node(&current_node, current_language.as_deref()),
179            };
180            let (from, to, nodes_for_replacement) =
181                get_diff_range(&current_node.get_children(), &highlight_nodes);
182            if from != to || !nodes_for_replacement.is_empty() {
183                node.splice(from, to - from, nodes_for_replacement)?;
184                return Ok(true);
185            }
186            Ok(false)
187        })
188    }
189
190    /// Adds `node_key` to the blocks waiting for `language`, and asks the
191    /// tokenizer to load it when nothing waited for it yet. Once it has
192    /// loaded, an update tagged `history-merge` marks each waiting block
193    /// that still has the language highlighted and dirty, so the transform
194    /// highlights it (`loadCodeLanguage`'s callback).
195    fn wait_for_language(self: &Rc<Self>, language: String, node_key: NodeKey) {
196        {
197            let mut state = self.state.borrow_mut();
198            if let Some((_, keys)) = state
199                .waiting_for_language
200                .iter_mut()
201                .find(|(l, _)| *l == language)
202            {
203                if !keys.contains(&node_key) {
204                    keys.push(node_key);
205                }
206                return;
207            }
208            state
209                .waiting_for_language
210                .push((language.clone(), vec![node_key]));
211        }
212        let this = Rc::downgrade(self);
213        let loaded = language.clone();
214        self.tokenizer.load_language(
215            &language,
216            Box::new(move || {
217                let Some(this) = this.upgrade() else {
218                    return;
219                };
220                let keys = {
221                    let mut state = this.state.borrow_mut();
222                    let index = state
223                        .waiting_for_language
224                        .iter()
225                        .position(|(l, _)| *l == loaded);
226                    index.map_or_else(Vec::new, |i| state.waiting_for_language.remove(i).1)
227                };
228                let Some(editor) = this.editor.upgrade().map(Editor) else {
229                    return;
230                };
231                let tokenizer = this.tokenizer.clone();
232                let _ = editor.update(
233                    move || {
234                        for key in keys {
235                            let Some(code_node) = get_node_by_key(&key).filter(is_code_node) else {
236                                continue;
237                            };
238                            if language_or_default(&code_node, &*tokenizer).as_deref()
239                                == Some(loaded.as_str())
240                            {
241                                code_node.set_is_syntax_highlight_supported(true);
242                                code_node.mark_dirty();
243                            }
244                        }
245                        Ok(())
246                    },
247                    UpdateOptions {
248                        tags: vec![HISTORY_MERGE_TAG.to_owned()],
249                        ..UpdateOptions::default()
250                    },
251                );
252            }),
253        );
254    }
255}
256
257/// `$updateAndRetainSelection(nodeKey, updateFn)`: runs `update_fn`, and
258/// when it changed the block, puts a caret that was in it back at the same
259/// text offset.
260fn update_and_retain_selection(
261    node_key: &str,
262    update_fn: impl FnOnce() -> Result<bool, LexicalError>,
263) -> Result<(), LexicalError> {
264    let Some(node) = get_node_by_key(node_key) else {
265        return Ok(());
266    };
267    if !is_code_node(&node) || !node.is_attached() {
268        return Ok(());
269    }
270    // Only a range selection inside this block is retained: restoring one
271    // elsewhere would drag the caret into the block.
272    let Some(selection) = get_selection().filter(|s| s.range().is_some()) else {
273        update_fn()?;
274        return Ok(());
275    };
276    let anchor = selection.point(Which::Anchor).expect("a range selection");
277    let anchor_node = anchor.get_node()?;
278    if anchor_node != node && !node.is_parent_of(&anchor_node) {
279        update_fn()?;
280        return Ok(());
281    }
282    let anchor_offset = anchor.offset;
283    let is_new_line_anchor = anchor.point_type == PointType::Element
284        && anchor_offset
285            .checked_sub(1)
286            .and_then(|i| node.get_child_at_index(i))
287            .is_some_and(|child| is_line_break_node(&child));
288    let mut text_offset = 0;
289    // The text before the anchor (the text of its node's previous
290    // siblings), and the anchor's own offset.
291    if !is_new_line_anchor {
292        text_offset = anchor_offset
293            + anchor_node
294                .get_previous_siblings()
295                .iter()
296                .map(NodeRef::get_text_content_size)
297                .sum::<usize>();
298    }
299
300    if !update_fn()? {
301        return Ok(());
302    }
303
304    // Only a line break puts the anchor on the element.
305    if is_new_line_anchor {
306        let anchor = selection.point(Which::Anchor).expect("a range selection");
307        anchor
308            .get_node()?
309            .select(Some(anchor_offset), Some(anchor_offset))?;
310        return Ok(());
311    }
312    // Walk to the text offset. A LineBreakNode takes one unit but holds no
313    // text point, so an offset that lands on one is an element point.
314    let children = node.get_children();
315    for (index, child) in children.iter().enumerate() {
316        if child.is_text() {
317            let text_content_size = child.get_text_content_size();
318            if text_content_size >= text_offset {
319                child.select(Some(text_offset), Some(text_offset))?;
320                return Ok(());
321            }
322            text_offset -= text_content_size;
323        } else if is_line_break_node(child) {
324            if text_offset == 0 {
325                node.select(Some(index), Some(index))?;
326                return Ok(());
327            }
328            text_offset -= 1;
329        }
330    }
331    node.select(Some(children.len()), Some(children.len()))?;
332    Ok(())
333}
334
335/// `getDiffRange(prevNodes, nextNodes)`: the range `from..to` of
336/// `prev_nodes` to replace with the part of `next_nodes` that differs, so
337/// that the children the new tokens match are kept.
338fn get_diff_range<'a>(
339    prev_nodes: &[NodeRef],
340    next_nodes: &'a [NodeRef],
341) -> (usize, usize, &'a [NodeRef]) {
342    let mut leading_match = 0;
343    while leading_match < prev_nodes.len() {
344        match next_nodes.get(leading_match) {
345            Some(next) if is_equal(&prev_nodes[leading_match], next) => leading_match += 1,
346            _ => break,
347        }
348    }
349    let prev_nodes_length = prev_nodes.len();
350    let next_nodes_length = next_nodes.len();
351    let max_trailing_match = prev_nodes_length.min(next_nodes_length) - leading_match;
352    let mut trailing_match = 0;
353    while trailing_match < max_trailing_match {
354        trailing_match += 1;
355        if !is_equal(
356            &prev_nodes[prev_nodes_length - trailing_match],
357            &next_nodes[next_nodes_length - trailing_match],
358        ) {
359            trailing_match -= 1;
360            break;
361        }
362    }
363    (
364        leading_match,
365        prev_nodes_length - trailing_match,
366        &next_nodes[leading_match..next_nodes_length - trailing_match],
367    )
368}
369
370/// `isEqual(nodeA, nodeB)`: CodeHighlightNodes of the same text and type,
371/// tabs and line breaks. Any other text node differs, so it becomes a
372/// token.
373fn is_equal(node_a: &NodeRef, node_b: &NodeRef) -> bool {
374    if is_code_highlight_node(node_a) && is_code_highlight_node(node_b) {
375        return node_a.get_text_content() == node_b.get_text_content()
376            && node_a.get_highlight_type() == node_b.get_highlight_type();
377    }
378    (is_tab_node(node_a) && is_tab_node(node_b))
379        || (is_line_break_node(node_a) && is_line_break_node(node_b))
380}
381
382/// `registerHighlightingOnly(editor, tokenizer)`: the highlighting
383/// transforms of `CodeNode`, `TextNode` and `CodeHighlightNode`, without
384/// the indent handlers of [`register_code_indentation`] and without
385/// `registerCodeGutter`, which an app that draws line numbers registers
386/// itself ([`super::register_code_gutter`]). Returns what unregisters them.
387pub fn register_highlighting_only(editor: &Editor, tokenizer: Rc<dyn Tokenizer>) -> Unregister {
388    let highlighter = Rc::new(Highlighter {
389        editor: editor.weak(),
390        tokenizer,
391        state: RefCell::default(),
392    });
393    let code = highlighter.clone();
394    let text = highlighter.clone();
395    let code_highlight = highlighter;
396    super::merge(vec![
397        editor.register_node_transform("code", move |node| code.code_node_transform(node)),
398        editor.register_node_transform("text", move |node| text.text_node_transform(node)),
399        editor.register_node_transform("code-highlight", move |node| {
400            code_highlight.text_node_transform(node)
401        }),
402    ])
403}
404
405/// `registerCodeHighlighting(editor, tokenizer)`: highlighting with
406/// `tokenizer` and the indent handlers of [`register_code_indentation`]
407/// (without a `tabSize`), for code that doesn't use
408/// `code_highlighter_extension`. Fails
409/// when the editor lacks `CodeNode` or `CodeHighlightNode`. Returns what
410/// unregisters both.
411pub fn register_code_highlighting(
412    editor: &Editor,
413    tokenizer: Rc<dyn Tokenizer>,
414) -> Result<Unregister, LexicalError> {
415    let nodes = editor.registered_nodes();
416    invariant!(
417        nodes.contains_key("code") && nodes.contains_key("code-highlight"),
418        "CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor"
419    );
420    Ok(super::merge(vec![
421        register_highlighting_only(editor, tokenizer),
422        register_code_indentation(editor, None, false),
423    ]))
424}
425
426#[cfg(feature = "extension")]
427pub use extension::*;
428
429#[cfg(feature = "extension")]
430mod extension {
431    use super::super::code_extension::{code_extension, code_indent_extension};
432    use super::super::code_node::DEFAULT_CODE_LANGUAGE;
433    use super::super::code_tokenizer::{PlainTokenizer, Tokenizer};
434    use super::register_highlighting_only;
435    use crate::Editor;
436    use crate::extension::named_signals_config;
437    use crate::extension::signals::effect;
438    use crate::extension_core::{
439        ExtensionBuildState, ExtensionRegisterState, LexicalExtension, define_extension,
440    };
441    use std::rc::Rc;
442
443    /// A [`Tokenizer`] in a config, compared by identity, as JS compares
444    /// objects.
445    #[derive(Clone)]
446    pub struct TokenizerRef(pub Rc<dyn Tokenizer>);
447
448    impl PartialEq for TokenizerRef {
449        fn eq(&self, other: &Self) -> bool {
450            Rc::ptr_eq(&self.0, &other.0)
451        }
452    }
453
454    named_signals_config! {
455        /// `CodePrismConfig` / `CodeShikiConfig`.
456        pub struct CodeHighlighterConfig;
457        /// `Partial<CodePrismConfig>`.
458        pub struct CodeHighlighterConfigOverrides;
459        /// `namedSignals(CodePrismConfig)`: the extension's output.
460        pub struct CodeHighlighterOutput {
461            /// `disabled`: the highlighter isn't registered. Flipping the
462            /// signal switches it on or off without rebuilding the editor.
463            disabled: bool,
464            /// `tokenizer`: the engine that tokenizes code.
465            tokenizer: TokenizerRef,
466        }
467    }
468
469    /// `CodePrismExtension`, over the configured tokenizer.
470    pub type CodeHighlighterExtensionType =
471        LexicalExtension<CodeHighlighterConfig, CodeHighlighterOutput, ()>;
472
473    fn make_code_highlighter_extension() -> Rc<CodeHighlighterExtensionType> {
474        define_extension(LexicalExtension {
475            build: Some(Rc::new(
476                |_: &Editor, config: &CodeHighlighterConfig, _: &ExtensionBuildState<()>| {
477                    Ok(CodeHighlighterOutput::of(config))
478                },
479            )),
480            dependencies: vec![code_extension().into(), code_indent_extension().into()],
481            register: Some(Rc::new(
482                |editor: &Editor,
483                 _: &CodeHighlighterConfig,
484                 state: &ExtensionRegisterState<(), CodeHighlighterOutput>| {
485                    let stores = state
486                        .get_output()
487                        .expect("CodeHighlighterExtension has a build");
488                    let editor = editor.weak();
489                    Ok(effect(move || {
490                        if stores.disabled.value() {
491                            return None;
492                        }
493                        let tokenizer = stores.tokenizer.value();
494                        let editor = editor.upgrade().map(Editor)?;
495                        Some(register_highlighting_only(&editor, tokenizer.0))
496                    }))
497                },
498            )),
499            // Swift's name: the extension is neither Prism's nor Shiki's.
500            ..LexicalExtension::new(
501                "@lexical/code-highlighter",
502                CodeHighlighterConfig {
503                    disabled: false,
504                    tokenizer: TokenizerRef(Rc::new(PlainTokenizer {
505                        default_language: Some(DEFAULT_CODE_LANGUAGE.to_owned()),
506                    })),
507                },
508            )
509        })
510    }
511
512    thread_local! {
513        static CODE_HIGHLIGHTER_EXTENSION: Rc<CodeHighlighterExtensionType> =
514            make_code_highlighter_extension();
515    }
516
517    /// `CodePrismExtension`, over the configured tokenizer (by default a
518    /// [`PlainTokenizer`] defaulting to `javascript`, since Prism isn't
519    /// ported): highlights code blocks with
520    /// [`register_highlighting_only`], unless `disabled`.
521    /// [`code_extension`] registers the nodes, and [`code_indent_extension`]
522    /// brings the indent handlers.
523    pub fn code_highlighter_extension() -> Rc<CodeHighlighterExtensionType> {
524        CODE_HIGHLIGHTER_EXTENSION.with(Rc::clone)
525    }
526}
527
528#[cfg(test)]
529mod tests {
530    use super::super::code_node::create_code_node;
531    use super::super::code_nodes;
532    use super::super::code_tokenizer::{CodeToken, Tokenizer};
533    use super::*;
534    use crate::{CreateEditorArgs, create_editor, get_root};
535    use std::cell::Cell;
536
537    /// A tokenizer whose one language, `lazy`, loads when it is asked to,
538    /// as Shiki's grammars do, and makes the whole block one `lazy` token.
539    #[derive(Default)]
540    struct LazyTokenizer {
541        loaded: Cell<bool>,
542        loads: Cell<usize>,
543        on_ready: RefCell<Option<Box<dyn FnOnce()>>>,
544    }
545
546    impl Tokenizer for LazyTokenizer {
547        fn default_language(&self) -> Option<&str> {
548            Some("lazy")
549        }
550        fn language_state(&self, language: &str) -> CodeLanguageState {
551            match (language, self.loaded.get()) {
552                ("lazy", true) => CodeLanguageState::Loaded,
553                ("lazy", false) => CodeLanguageState::Loading,
554                _ => CodeLanguageState::Unsupported,
555            }
556        }
557        fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
558            vec![CodeToken::token("lazy", code)]
559        }
560        fn load_language(&self, _language: &str, on_ready: Box<dyn FnOnce()>) {
561            self.loads.set(self.loads.get() + 1);
562            *self.on_ready.borrow_mut() = Some(on_ready);
563        }
564    }
565
566    fn update(editor: &Editor, f: impl FnOnce() -> Result<(), LexicalError> + 'static) {
567        editor
568            .update(
569                f,
570                UpdateOptions {
571                    discrete: true,
572                    ..UpdateOptions::default()
573                },
574            )
575            .unwrap();
576    }
577
578    /// A child's type, text and highlight type.
579    type Child = (String, String, Option<String>);
580
581    /// Whether each block is highlighted, and its children.
582    fn blocks(editor: &Editor) -> Vec<(bool, Vec<Child>)> {
583        editor
584            .read(|| {
585                get_root()
586                    .get_children()
587                    .iter()
588                    .map(|block| {
589                        let children = block
590                            .get_children()
591                            .iter()
592                            .map(|c| {
593                                (
594                                    c.get_type().to_owned(),
595                                    c.get_text_content(),
596                                    c.get_highlight_type(),
597                                )
598                            })
599                            .collect();
600                        (block.get_is_syntax_highlight_supported(), children)
601                    })
602                    .collect()
603            })
604            .unwrap()
605    }
606
607    /// A language that is loading leaves its blocks alone; once it has
608    /// loaded (once, for both blocks), they are highlighted.
609    #[test]
610    fn a_loading_language_highlights_once_loaded() {
611        let editor = create_editor(CreateEditorArgs {
612            nodes: code_nodes(),
613            ..CreateEditorArgs::default()
614        });
615        let tokenizer = Rc::new(LazyTokenizer::default());
616        let _unregister = register_code_highlighting(&editor, tokenizer.clone()).unwrap();
617        update(&editor, || {
618            for text in ["a", "b"] {
619                let code = create_code_node(None, None);
620                code.append(&[create_text_node(text)])?;
621                get_root().append(&[code])?;
622            }
623            Ok(())
624        });
625        let plain = |text: &str| vec![("text".to_owned(), text.to_owned(), None)];
626        assert_eq!(
627            blocks(&editor),
628            vec![(false, plain("a")), (false, plain("b"))]
629        );
630        assert_eq!(tokenizer.loads.get(), 1);
631        tokenizer.loaded.set(true);
632        let on_ready = tokenizer.on_ready.borrow_mut().take().unwrap();
633        on_ready();
634        editor.read(|| ()).unwrap();
635        let lazy = |text: &str| {
636            vec![(
637                "code-highlight".to_owned(),
638                text.to_owned(),
639                Some("lazy".to_owned()),
640            )]
641        };
642        assert_eq!(blocks(&editor), vec![(true, lazy("a")), (true, lazy("b"))]);
643    }
644
645    /// `register_code_highlighting` needs the code nodes.
646    #[test]
647    fn registering_without_the_code_nodes_fails() {
648        let editor = create_editor(CreateEditorArgs::default());
649        let error = register_code_highlighting(&editor, Rc::new(LazyTokenizer::default()))
650            .err()
651            .unwrap();
652        assert_eq!(
653            error.message(),
654            "CodeHighlightPlugin: CodeNode or CodeHighlightNode not registered on editor"
655        );
656    }
657
658    /// `CodePrismExtension`'s `tokenizer` highlights, and its `disabled`
659    /// signal unregisters the highlighter.
660    #[cfg(feature = "extension")]
661    #[test]
662    fn the_extension_highlights_unless_disabled() {
663        use crate::extension::{
664            build_editor_from_extensions, get_extension_dependency_from_editor,
665        };
666        use crate::extension_core::config_extension;
667
668        struct Words;
669        impl Tokenizer for Words {
670            fn default_language(&self) -> Option<&str> {
671                Some("words")
672            }
673            fn language_state(&self, _language: &str) -> CodeLanguageState {
674                CodeLanguageState::Loaded
675            }
676            fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
677                vec![CodeToken::token("word", code)]
678            }
679        }
680        let built = build_editor_from_extensions(vec![config_extension(
681            &code_highlighter_extension(),
682            [CodeHighlighterConfigOverrides {
683                tokenizer: Some(TokenizerRef(Rc::new(Words))),
684                ..CodeHighlighterConfigOverrides::default()
685            }],
686        )])
687        .unwrap();
688        let editor = built.editor().clone();
689        editor.read(|| ()).unwrap();
690        update(&editor, || {
691            let code = create_code_node(None, None);
692            code.append(&[create_text_node("a")])?;
693            get_root().clear()?;
694            get_root().append(&[code])
695        });
696        let word = (
697            "code-highlight".to_owned(),
698            "a".to_owned(),
699            Some("word".to_owned()),
700        );
701        assert_eq!(blocks(&editor), vec![(true, vec![word.clone()])]);
702        let output = get_extension_dependency_from_editor(&editor, &code_highlighter_extension())
703            .unwrap()
704            .output
705            .unwrap();
706        output.disabled.set_value(true);
707        update(&editor, || {
708            let code = get_root().get_first_child().unwrap();
709            code.append(&[create_text_node("b")])
710        });
711        let text = ("text".to_owned(), "b".to_owned(), None);
712        assert_eq!(blocks(&editor), vec![(true, vec![word, text])]);
713    }
714}