Skip to main content

compose_rust/
highlight.rs

1//! From a highlighter's names to the code colour roles.
2//!
3//! A highlighter, tree-sitter or anything that speaks its vocabulary, names each token with
4//! a dotted capture name such as `keyword.return` or `variable.parameter`. The Renderer does
5//! not know any language and never hears these names: they are turned into colour roles
6//! here, on the Host side, and the roles travel as the `Paint` of a span the way any other
7//! coloured run does. Markdown is handled the same way, for the same reason.
8//!
9//! The names are matched from the most specific prefix down, so `string.escape` is an
10//! escape while `string.special` is still a string, and a name nothing here recognises gives
11//! `None`. A token without a role is left in the body ink rather than painted a guess.
12
13use crate::schema::{ColorRole, Paint};
14use crate::spans::TextSpan;
15
16/// The colour role a highlight name is painted with, or None to leave it in the body ink.
17///
18/// Accepts tree-sitter's capture names (`keyword.control`, `function.method`,
19/// `punctuation.bracket`, `diff.plus`), the older names some grammars still use
20/// (`conditional`, `repeat`, `include`, `float`, `field`, `preproc`), and VS Code's
21/// semantic token kinds (`enumMember`, `typeParameter`, `decorator`).
22pub fn role_for_highlight(name: &str) -> Option<ColorRole> {
23    // Longest first: every row is a prefix that ends at a dot or at the end of the name.
24    const TABLE: &[(&str, ColorRole)] = &[
25        ("string.escape", ColorRole::SyntaxEscape),
26        ("string.regexp", ColorRole::SyntaxEscape),
27        ("string.regex", ColorRole::SyntaxEscape),
28        ("constant.character.escape", ColorRole::SyntaxEscape),
29        ("function.macro", ColorRole::SyntaxMacro),
30        ("keyword.directive", ColorRole::SyntaxMacro),
31        ("keyword.import", ColorRole::SyntaxKeyword),
32        ("variable.member", ColorRole::SyntaxProperty),
33        ("variable.other.member", ColorRole::SyntaxProperty),
34        ("tag.attribute", ColorRole::SyntaxAttribute),
35        ("constructor", ColorRole::SyntaxFunction),
36        ("keyword", ColorRole::SyntaxKeyword),
37        ("conditional", ColorRole::SyntaxKeyword),
38        ("repeat", ColorRole::SyntaxKeyword),
39        ("exception", ColorRole::SyntaxKeyword),
40        ("include", ColorRole::SyntaxKeyword),
41        ("storageclass", ColorRole::SyntaxKeyword),
42        ("modifier", ColorRole::SyntaxKeyword),
43        ("string", ColorRole::SyntaxString),
44        ("character", ColorRole::SyntaxString),
45        ("comment", ColorRole::SyntaxComment),
46        ("number", ColorRole::SyntaxNumber),
47        ("float", ColorRole::SyntaxNumber),
48        ("boolean", ColorRole::SyntaxConstant),
49        ("constant", ColorRole::SyntaxConstant),
50        ("enumMember", ColorRole::SyntaxConstant),
51        ("type", ColorRole::SyntaxType),
52        ("typeParameter", ColorRole::SyntaxType),
53        ("class", ColorRole::SyntaxType),
54        ("struct", ColorRole::SyntaxType),
55        ("enum", ColorRole::SyntaxType),
56        ("interface", ColorRole::SyntaxType),
57        ("namespace", ColorRole::SyntaxType),
58        ("module", ColorRole::SyntaxType),
59        ("function", ColorRole::SyntaxFunction),
60        ("method", ColorRole::SyntaxFunction),
61        ("variable", ColorRole::SyntaxVariable),
62        ("parameter", ColorRole::SyntaxVariable),
63        ("property", ColorRole::SyntaxProperty),
64        ("field", ColorRole::SyntaxProperty),
65        ("operator", ColorRole::SyntaxOperator),
66        ("punctuation", ColorRole::SyntaxPunctuation),
67        ("tag", ColorRole::SyntaxTag),
68        ("attribute", ColorRole::SyntaxAttribute),
69        ("macro", ColorRole::SyntaxMacro),
70        ("preproc", ColorRole::SyntaxMacro),
71        ("define", ColorRole::SyntaxMacro),
72        ("decorator", ColorRole::SyntaxMacro),
73        ("annotation", ColorRole::SyntaxMacro),
74        ("regexp", ColorRole::SyntaxEscape),
75        ("diff.plus", ColorRole::DiffAdded),
76        ("diff.minus", ColorRole::DiffRemoved),
77        ("diff.delta", ColorRole::DiffModified),
78    ];
79    let matches = |prefix: &str| {
80        name.strip_prefix(prefix)
81            .is_some_and(|rest| rest.is_empty() || rest.starts_with('.'))
82    };
83    TABLE
84        .iter()
85        .filter(|(prefix, _)| matches(prefix))
86        .max_by_key(|(prefix, _)| prefix.len())
87        .map(|(_, role)| *role)
88}
89
90/// A span painting `length` bytes from `start` in the role a highlight name maps to, or
91/// None where the name has no role and the run is left in the body ink.
92pub fn highlight_span(start: u32, length: u32, name: &str) -> Option<TextSpan> {
93    role_for_highlight(name).map(|role| TextSpan::new(start, length).with_color(Paint::Role(role)))
94}
95
96#[cfg(test)]
97mod tests {
98    use super::*;
99
100    /// The finer names fold into the twenty two, and the most specific prefix wins.
101    #[test]
102    fn fr13_1_3_highlight_names_fold_into_the_code_roles() {
103        for (name, role) in [
104            ("keyword", ColorRole::SyntaxKeyword),
105            ("keyword.return", ColorRole::SyntaxKeyword),
106            ("keyword.directive", ColorRole::SyntaxMacro),
107            ("string", ColorRole::SyntaxString),
108            ("string.special", ColorRole::SyntaxString),
109            ("string.escape", ColorRole::SyntaxEscape),
110            ("string.regexp", ColorRole::SyntaxEscape),
111            ("character", ColorRole::SyntaxString),
112            ("comment.documentation", ColorRole::SyntaxComment),
113            ("number", ColorRole::SyntaxNumber),
114            ("float", ColorRole::SyntaxNumber),
115            ("constant.builtin", ColorRole::SyntaxConstant),
116            ("boolean", ColorRole::SyntaxConstant),
117            ("type.builtin", ColorRole::SyntaxType),
118            ("function.method", ColorRole::SyntaxFunction),
119            ("function.macro", ColorRole::SyntaxMacro),
120            ("constructor", ColorRole::SyntaxFunction),
121            ("variable.parameter", ColorRole::SyntaxVariable),
122            ("variable.member", ColorRole::SyntaxProperty),
123            ("property", ColorRole::SyntaxProperty),
124            ("field", ColorRole::SyntaxProperty),
125            ("operator", ColorRole::SyntaxOperator),
126            ("punctuation.bracket", ColorRole::SyntaxPunctuation),
127            ("tag", ColorRole::SyntaxTag),
128            ("attribute", ColorRole::SyntaxAttribute),
129            ("preproc", ColorRole::SyntaxMacro),
130            ("diff.plus", ColorRole::DiffAdded),
131            ("diff.minus", ColorRole::DiffRemoved),
132            ("diff.delta", ColorRole::DiffModified),
133        ] {
134            assert_eq!(role_for_highlight(name), Some(role), "{name}");
135        }
136    }
137
138    /// A name nothing here knows stays in the body ink, and a prefix only counts at a dot.
139    #[test]
140    fn fr13_1_3_an_unknown_name_is_left_unpainted() {
141        assert_eq!(role_for_highlight("embedded"), None);
142        assert_eq!(role_for_highlight("spell"), None);
143        assert_eq!(role_for_highlight("keywords"), None);
144        assert!(highlight_span(0, 3, "none").is_none());
145    }
146
147    /// Highlighting a Rust snippet into a `Text` sends every coloured run as a colour role on
148    /// the wire and never stamps a literal ARGB colour into the batch.
149    #[test]
150    fn fr13_1_3_highlighted_rust_text_carries_no_literal_colour_on_the_wire() {
151        use crate::protocol::{Mutation, PropertyValue, decode_batch};
152        use crate::schema::{Modifier as WireModifier, PropertyKind};
153        use crate::spans::{SPAN_COLOR_AT, SPAN_LEN, TextSpans};
154
155        let source = "pub fn total(items: &[u32]) -> u32 { // forty-two\n    items.len() + 42\n}";
156        let tokens = [
157            ("pub", "keyword", Some(ColorRole::SyntaxKeyword)),
158            ("fn", "keyword", Some(ColorRole::SyntaxKeyword)),
159            ("total", "function", Some(ColorRole::SyntaxFunction)),
160            (
161                "items",
162                "variable.parameter",
163                Some(ColorRole::SyntaxVariable),
164            ),
165            ("&", "operator", Some(ColorRole::SyntaxOperator)),
166            (
167                "[",
168                "punctuation.bracket",
169                Some(ColorRole::SyntaxPunctuation),
170            ),
171            ("u32", "type.builtin", Some(ColorRole::SyntaxType)),
172            (
173                "]",
174                "punctuation.bracket",
175                Some(ColorRole::SyntaxPunctuation),
176            ),
177            ("->", "operator", Some(ColorRole::SyntaxOperator)),
178            ("// forty-two", "comment", Some(ColorRole::SyntaxComment)),
179            ("items", "spell", None),
180            ("len", "function.method", Some(ColorRole::SyntaxFunction)),
181            ("+", "operator", Some(ColorRole::SyntaxOperator)),
182            ("42", "number", Some(ColorRole::SyntaxNumber)),
183        ];
184        let mut cursor = 0;
185        let mut spans = Vec::new();
186        let mut expected_spans = Vec::new();
187        for (lexeme, capture, expected_role) in tokens {
188            let relative = source[cursor..]
189                .find(lexeme)
190                .unwrap_or_else(|| panic!("missing token {lexeme} after byte {cursor}"));
191            let start = (cursor + relative) as u32;
192            let length = lexeme.len() as u32;
193            cursor = start as usize + lexeme.len();
194            let span = highlight_span(start, length, capture);
195            assert_eq!(
196                span.as_ref().and_then(|s| s.color),
197                expected_role.map(Paint::Role),
198                "unexpected paint for {lexeme} ({capture})"
199            );
200            if let Some(span) = span {
201                expected_spans.push((lexeme, expected_role.unwrap()));
202                spans.push(span);
203            }
204        }
205        assert_eq!(spans.len(), 13);
206
207        let mut host = crate::ComposeHost::with_content(move || {
208            let encoded_spans = TextSpans::new(spans.clone());
209            crate::ui::Text(source).spans(encoded_spans);
210        });
211        let wire = host.rebuild().expect("the batch encodes").to_vec();
212        let mutations = decode_batch(&wire).expect("the batch decodes");
213
214        let mut blobs = Vec::new();
215        for mutation in &mutations {
216            match mutation {
217                Mutation::SetProp {
218                    property: PropertyKind::Spans,
219                    value: PropertyValue::Bytes(bytes),
220                    ..
221                } => {
222                    blobs.push(*bytes);
223                }
224                Mutation::SetProp {
225                    property: PropertyKind::Color,
226                    value: PropertyValue::Integer(bits),
227                    ..
228                } => {
229                    assert_ne!(
230                        Paint::from_bits(*bits as u64),
231                        None,
232                        "any colour property on the wire must decode as a valid paint"
233                    );
234                    assert!(
235                        !matches!(Paint::from_bits(*bits as u64), Some(Paint::Literal(_))),
236                        "a highlighted Text must not send a literal node colour"
237                    );
238                }
239                Mutation::SetModifier {
240                    modifier: WireModifier::Background(paint),
241                    ..
242                } => {
243                    assert!(
244                        !matches!(paint, Paint::Literal(_)),
245                        "a highlighted Text must not send a literal background"
246                    );
247                }
248                _ => {}
249            }
250        }
251
252        assert_eq!(blobs.len(), 1, "one text, one list of runs: {mutations:?}");
253        let span_bytes = blobs[0];
254        assert_eq!(span_bytes.len(), expected_spans.len() * SPAN_LEN);
255
256        for (record, (lexeme, _)) in span_bytes.chunks_exact(SPAN_LEN).zip(&expected_spans) {
257            let start = u32::from_le_bytes([record[0], record[1], record[2], record[3]]) as usize;
258            let len = u32::from_le_bytes([record[4], record[5], record[6], record[7]]) as usize;
259            assert_eq!(&source[start..start + len], *lexeme);
260            let paint_kind = u32::from_le_bytes([
261                record[SPAN_COLOR_AT + 4],
262                record[SPAN_COLOR_AT + 5],
263                record[SPAN_COLOR_AT + 6],
264                record[SPAN_COLOR_AT + 7],
265            ]);
266            assert_eq!(
267                paint_kind, 1,
268                "every highlighted run must carry Paint::Role (tag 1) on the wire, never a literal colour"
269            );
270        }
271
272        let decoded: Vec<_> = TextSpans::from_bytes(span_bytes).spans().collect();
273        assert_eq!(decoded.len(), expected_spans.len());
274        for (span, (lexeme, role)) in decoded.iter().zip(&expected_spans) {
275            assert_eq!(
276                span.color,
277                Some(Paint::Role(*role)),
278                "decoded span for {lexeme} must carry role {role:?}"
279            );
280            assert!(
281                span.background.is_none(),
282                "syntax highlighting sets ink roles only, got {:?}",
283                span.background
284            );
285        }
286    }
287
288    /// Adding the twenty-two code colour roles extends `ColorRole` and the token tables
289    /// without altering the widget definitions, the property schema or the modifier schema.
290    #[test]
291    fn fr13_1_3_code_roles_leave_widget_property_and_modifier_schemas_unchanged() {
292        use crate::protocol::{BatchEncoder, Mutation, PropertyValue};
293        use crate::schema::{
294            COLOR_ROLE_COUNT, COLOR_ROLE_SCHEMA, EnumVariantSchema, FieldSlot, FieldType,
295            MODIFIER_SCHEMA, Modifier, PROPERTY_SCHEMA, PropertyKind, SCHEMA_DESCRIPTOR,
296            WIDGET_SCHEMA,
297        };
298
299        assert_eq!(COLOR_ROLE_COUNT, 45);
300        assert_eq!(COLOR_ROLE_SCHEMA.len(), COLOR_ROLE_COUNT);
301        assert_eq!(COLOR_ROLE_SCHEMA[22].name, "OnTertiaryContainer");
302        assert_eq!(COLOR_ROLE_SCHEMA[22].tag, 23);
303        assert_eq!(COLOR_ROLE_SCHEMA[23].name, "SyntaxKeyword");
304        assert_eq!(COLOR_ROLE_SCHEMA[23].tag, 24);
305        assert_eq!(COLOR_ROLE_SCHEMA[44].name, "DiffRemovedEmphasis");
306        assert_eq!(COLOR_ROLE_SCHEMA[44].tag, 45);
307        for (index, entry) in COLOR_ROLE_SCHEMA.iter().enumerate() {
308            assert_eq!(
309                entry.tag as usize,
310                index + 1,
311                "colour roles are append-only with contiguous tags"
312            );
313        }
314
315        let hash_bytes = |mut hash: u64, bytes: &[u8]| -> u64 {
316            for &byte in bytes {
317                hash ^= byte as u64;
318                hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
319            }
320            hash
321        };
322        let hash_enum = |mut hash: u64, schema: &[EnumVariantSchema]| -> u64 {
323            for entry in schema {
324                hash = hash_bytes(hash, entry.name.as_bytes());
325                hash = hash_bytes(hash, &entry.tag.to_le_bytes());
326            }
327            hash
328        };
329        let mut tables_hash = 0xcbf2_9ce4_8422_2325_u64;
330        tables_hash = hash_enum(tables_hash, WIDGET_SCHEMA);
331        tables_hash = hash_enum(tables_hash, PROPERTY_SCHEMA);
332        for modifier in MODIFIER_SCHEMA {
333            tables_hash = hash_bytes(tables_hash, modifier.name.as_bytes());
334            tables_hash = hash_bytes(tables_hash, &modifier.tag.to_le_bytes());
335            tables_hash = hash_bytes(tables_hash, &[modifier.extra_words]);
336            for field in modifier.fields {
337                tables_hash = hash_bytes(tables_hash, field.name.as_bytes());
338                tables_hash = hash_bytes(
339                    tables_hash,
340                    &[match field.ty {
341                        FieldType::Float => 1,
342                        FieldType::U32 => 2,
343                        FieldType::U64 => 3,
344                        FieldType::Paint => 4,
345                        FieldType::Role(_) => 5,
346                        FieldType::Bool => 6,
347                        FieldType::FiniteFloat => 7,
348                    }],
349                );
350                if let FieldType::Role(role) = field.ty {
351                    tables_hash = hash_bytes(tables_hash, role.as_bytes());
352                }
353                tables_hash = hash_bytes(
354                    tables_hash,
355                    &[match field.slot {
356                        FieldSlot::FirstLow => 0,
357                        FieldSlot::FirstHigh => 1,
358                        FieldSlot::First => 2,
359                        FieldSlot::SecondLow => 3,
360                        FieldSlot::SecondHigh => 4,
361                        FieldSlot::Second => 5,
362                        FieldSlot::ThirdLow => 6,
363                        FieldSlot::ThirdHigh => 7,
364                        FieldSlot::Third => 8,
365                        FieldSlot::FourthLow => 9,
366                        FieldSlot::FourthHigh => 10,
367                        FieldSlot::Fourth => 11,
368                        FieldSlot::FifthLow => 12,
369                        FieldSlot::FifthHigh => 13,
370                        FieldSlot::Fifth => 14,
371                        FieldSlot::SixthLow => 15,
372                        FieldSlot::SixthHigh => 16,
373                        FieldSlot::Sixth => 17,
374                        FieldSlot::SeventhLow => 18,
375                        FieldSlot::SeventhHigh => 19,
376                        FieldSlot::Seventh => 20,
377                        FieldSlot::EighthLow => 21,
378                        FieldSlot::EighthHigh => 22,
379                        FieldSlot::Eighth => 23,
380                        FieldSlot::NinthLow => 24,
381                        FieldSlot::NinthHigh => 25,
382                        FieldSlot::Ninth => 26,
383                        FieldSlot::TenthLow => 27,
384                        FieldSlot::TenthHigh => 28,
385                        FieldSlot::Tenth => 29,
386                    }],
387                );
388            }
389        }
390        // A recorded value, not a rule: a change that means to add a widget, a property or a
391        // modifier updates the numbers below in the same commit. What this catches is a
392        // change that moves them without meaning to, as adding a colour role must not.
393        let on_purpose = "if this change adds or alters a widget, a property or a modifier on \
394                          purpose, record the new value here; a change to colour roles alone \
395                          must leave it as it is";
396        assert_eq!(
397            WIDGET_SCHEMA.len(),
398            49,
399            "the number of widgets moved: {on_purpose}"
400        );
401        assert_eq!(
402            PROPERTY_SCHEMA.len(),
403            151,
404            "the number of properties moved: {on_purpose}"
405        );
406        assert_eq!(
407            MODIFIER_SCHEMA.len(),
408            40,
409            "the number of modifiers moved: {on_purpose}"
410        );
411        assert_eq!(
412            tables_hash, 0xb9d6_3156_0d1e_c3f4,
413            "the names, tags or modifier fields of the widget, property and modifier tables \
414             moved, and now hash to {tables_hash:#018x}: {on_purpose}"
415        );
416
417        let folded = |name: &str| -> String {
418            name.chars()
419                .filter(|ch| *ch != '_')
420                .map(|ch| ch.to_ascii_lowercase())
421                .collect()
422        };
423        let mut entries = Vec::new();
424        for section in SCHEMA_DESCRIPTOR.split(';') {
425            if let Some(rest) = section.strip_prefix("widgets=") {
426                for name in rest.split(',') {
427                    entries.push(("widgets", name));
428                }
429            } else if let Some(rest) = section.strip_prefix("properties=") {
430                for name in rest.split(',') {
431                    entries.push(("properties", name));
432                }
433            } else if let Some(rest) = section.strip_prefix("modifiers=") {
434                for name in rest.split(',') {
435                    entries.push(("modifiers", name));
436                }
437            }
438        }
439        for widget in WIDGET_SCHEMA {
440            entries.push(("widgets", widget.name));
441        }
442        for property in PROPERTY_SCHEMA {
443            entries.push(("properties", property.name));
444        }
445        for modifier in MODIFIER_SCHEMA {
446            entries.push(("modifiers", modifier.name));
447            for field in modifier.fields {
448                entries.push(("modifiers", field.name));
449            }
450        }
451        let about_code: Vec<_> = entries
452            .iter()
453            .copied()
454            .filter(|(_, name)| {
455                let name = folded(name);
456                (name.contains("syntax") || name.contains("diff")) && name != "syntaxspans"
457            })
458            .collect();
459        assert_eq!(about_code, Vec::<(&str, &str)>::new());
460
461        for modifier in MODIFIER_SCHEMA {
462            for field in modifier.fields {
463                assert_ne!(
464                    field.ty,
465                    FieldType::Role("ColorRole"),
466                    "modifier {} field {} should take a Paint rather than a dedicated ColorRole slot",
467                    modifier.name,
468                    field.name
469                );
470            }
471        }
472
473        let encode_one = |mutation: Mutation<'_>| -> Vec<u8> {
474            let mut encoder = BatchEncoder::default();
475            encoder.encode(&mutation).expect("encodes");
476            encoder.finish().expect("finishes").to_vec()
477        };
478        let base_prop = encode_one(Mutation::SetProp {
479            node_id: 1,
480            property: PropertyKind::Color,
481            value: PropertyValue::Integer(Paint::Role(ColorRole::Primary).to_bits() as i64),
482        });
483        let code_prop = encode_one(Mutation::SetProp {
484            node_id: 1,
485            property: PropertyKind::Color,
486            value: PropertyValue::Integer(Paint::Role(ColorRole::SyntaxKeyword).to_bits() as i64),
487        });
488        assert_eq!(base_prop.len(), 36);
489        assert_eq!(code_prop.len(), 36);
490
491        let base_mod = encode_one(Mutation::SetModifier {
492            node_id: 1,
493            index: 0,
494            modifier: Modifier::Background(Paint::Role(ColorRole::SurfaceContainer)),
495        });
496        let code_mod = encode_one(Mutation::SetModifier {
497            node_id: 1,
498            index: 0,
499            modifier: Modifier::Background(Paint::Role(ColorRole::DiffAddedContainer)),
500        });
501        assert_eq!(base_mod.len(), 40);
502        assert_eq!(code_mod.len(), 40);
503    }
504}