packdiff_dto/
highlight.rs

1//! Safe, dependency-free lexical syntax highlighting.
2//!
3//! One generic scanner is parameterized by static language profiles. The
4//! output is safe HTML: source text is escaped here, and the only markup this
5//! module emits is its own fixed `tok-*` spans.
6
7use crate::diff::Line;
8
9#[derive(Clone, Copy)]
10struct StringKind {
11  delim: &'static str,
12  escapes: bool,
13  multiline: bool,
14}
15
16struct Profile {
17  line_comments: &'static [&'static str],
18  block_comment: Option<(&'static str, &'static str)>,
19  strings: &'static [StringKind],
20  keywords: &'static [&'static str],
21  literals: &'static [&'static str],
22  fn_call: bool,
23}
24
25#[derive(Clone, Copy)]
26enum State {
27  Neutral,
28  BlockComment(&'static str),
29  String(StringKind),
30}
31
32struct Scanner {
33  profile: &'static Profile,
34  state: State,
35}
36
37impl Scanner {
38  fn new(profile: &'static Profile) -> Self {
39    Self { profile, state: State::Neutral }
40  }
41
42  fn line(&mut self, input: &str) -> String {
43    let mut out = String::with_capacity(input.len());
44    let mut pos = 0;
45    while pos < input.len() {
46      match self.state {
47        State::BlockComment(end) => {
48          let close = input[pos..].find(end).map(|i| pos + i + end.len());
49          let stop = close.unwrap_or(input.len());
50          token(&mut out, "tok-com", &input[pos..stop]);
51          pos = stop;
52          if close.is_some() {
53            self.state = State::Neutral;
54          }
55        }
56        State::String(kind) => {
57          let (stop, closed) = string_end(input, pos, kind, false);
58          token(&mut out, "tok-str", &input[pos..stop]);
59          pos = stop;
60          if closed {
61            self.state = State::Neutral;
62          }
63        }
64        State::Neutral => {
65          if self.profile.line_comments.iter().any(|marker| input[pos..].starts_with(*marker)) {
66            token(&mut out, "tok-com", &input[pos..]);
67            break;
68          }
69          if let Some((start, end)) = self.profile.block_comment.filter(|(start, _)| input[pos..].starts_with(start)) {
70            let close = input[pos + start.len()..].find(end).map(|i| pos + start.len() + i + end.len());
71            let stop = close.unwrap_or(input.len());
72            token(&mut out, "tok-com", &input[pos..stop]);
73            pos = stop;
74            if close.is_none() {
75              self.state = State::BlockComment(end);
76            }
77            continue;
78          }
79          if let Some(kind) = self.profile.strings.iter().find(|kind| input[pos..].starts_with(kind.delim)).copied() {
80            let (stop, closed) = string_end(input, pos, kind, true);
81            token(&mut out, "tok-str", &input[pos..stop]);
82            pos = stop;
83            if !closed && kind.multiline {
84              self.state = State::String(kind);
85            }
86            continue;
87          }
88          let ch = input[pos..].chars().next().expect("pos is within the input");
89          if ch.is_ascii_digit() || (ch == '.' && input[pos + 1..].starts_with(|c: char| c.is_ascii_digit())) {
90            let stop = number_end(input, pos);
91            token(&mut out, "tok-num", &input[pos..stop]);
92            pos = stop;
93            continue;
94          }
95          if ident_start(ch) {
96            let stop = ident_end(input, pos);
97            let word = &input[pos..stop];
98            let class = if self.profile.keywords.binary_search(&word).is_ok() {
99              Some("tok-kw")
100            } else if self.profile.literals.binary_search(&word).is_ok() {
101              Some("tok-lit")
102            } else if self.profile.fn_call && input[stop..].trim_start().starts_with('(') {
103              Some("tok-fn")
104            } else {
105              None
106            };
107            if let Some(class) = class {
108              token(&mut out, class, word);
109            } else {
110              escape_into(&mut out, word);
111            }
112            pos = stop;
113            continue;
114          }
115          escape_char(&mut out, ch);
116          pos += ch.len_utf8();
117        }
118      }
119    }
120    out
121  }
122}
123
124fn string_end(input: &str, start: usize, kind: StringKind, opening: bool) -> (usize, bool) {
125  let mut pos = start + if opening { kind.delim.len() } else { 0 };
126  let mut escaped = false;
127  while pos < input.len() {
128    if !escaped && input[pos..].starts_with(kind.delim) {
129      return (pos + kind.delim.len(), true);
130    }
131    let ch = input[pos..].chars().next().expect("pos is within the input");
132    escaped = kind.escapes && ch == '\\' && !escaped;
133    pos += ch.len_utf8();
134  }
135  (input.len(), false)
136}
137
138fn ident_start(ch: char) -> bool {
139  ch == '_' || ch.is_alphabetic()
140}
141
142fn ident_end(input: &str, start: usize) -> usize {
143  input[start..]
144    .char_indices()
145    .find(|(i, ch)| *i > 0 && !(*ch == '_' || ch.is_alphanumeric()))
146    .map_or(input.len(), |(i, _)| start + i)
147}
148
149fn number_end(input: &str, start: usize) -> usize {
150  let mut end = start;
151  let mut previous = '\0';
152  for (i, ch) in input[start..].char_indices() {
153    let allowed = ch.is_ascii_alphanumeric()
154      || matches!(ch, '_' | '.')
155      || (matches!(ch, '+' | '-') && matches!(previous, 'e' | 'E' | 'p' | 'P'));
156    if !allowed {
157      break;
158    }
159    end = start + i + ch.len_utf8();
160    previous = ch;
161  }
162  end
163}
164
165fn token(out: &mut String, class: &str, text: &str) {
166  out.push_str("<span class=\"");
167  out.push_str(class);
168  out.push_str("\">");
169  escape_into(out, text);
170  out.push_str("</span>");
171}
172
173fn escape_into(out: &mut String, text: &str) {
174  for ch in text.chars() {
175    escape_char(out, ch);
176  }
177}
178
179fn escape_char(out: &mut String, ch: char) {
180  match ch {
181    '&' => out.push_str("&amp;"),
182    '<' => out.push_str("&lt;"),
183    '>' => out.push_str("&gt;"),
184    '"' => out.push_str("&quot;"),
185    '\'' => out.push_str("&#39;"),
186    _ => out.push(ch),
187  }
188}
189
190/// Highlight a contiguous run of source lines. Unknown paths return `None`,
191/// allowing callers to preserve their plain escaped rendering.
192pub fn highlight_lines(path: &str, lines: &[&str]) -> Option<Vec<String>> {
193  let mut scanner = Scanner::new(profile_for(path)?);
194  Some(lines.iter().map(|line| scanner.line(line)).collect())
195}
196
197/// Highlight a diff hunk with independent lexer state for its old and new
198/// sides. Context advances both streams and renders from the new side.
199pub fn highlight_hunk(path: &str, lines: &[Line]) -> Option<Vec<String>> {
200  let profile = profile_for(path)?;
201  let mut old = Scanner::new(profile);
202  let mut new = Scanner::new(profile);
203  Some(
204    lines
205      .iter()
206      .map(|line| match line {
207        Line::Add { text, .. } => new.line(text),
208        Line::Del { text, .. } => old.line(text),
209        Line::Ctx { text, .. } => {
210          old.line(text);
211          new.line(text)
212        }
213        Line::Meta { text } => {
214          let mut html = String::new();
215          escape_into(&mut html, text);
216          html
217        }
218      })
219      .collect(),
220  )
221}
222
223const DQ: StringKind = StringKind { delim: "\"", escapes: true, multiline: false };
224const SQ: StringKind = StringKind { delim: "'", escapes: true, multiline: false };
225const RAW: StringKind = StringKind { delim: "`", escapes: false, multiline: true };
226const TRIPLE_DQ: StringKind = StringKind { delim: "\"\"\"", escapes: true, multiline: true };
227const TRIPLE_SQ: StringKind = StringKind { delim: "'''", escapes: true, multiline: true };
228
229const C_LIKE: Profile = Profile {
230  line_comments: &["//"],
231  block_comment: Some(("/*", "*/")),
232  strings: &[DQ, SQ],
233  keywords: &[
234    "alignas",
235    "alignof",
236    "asm",
237    "auto",
238    "bool",
239    "break",
240    "case",
241    "catch",
242    "char",
243    "class",
244    "const",
245    "constexpr",
246    "continue",
247    "default",
248    "delete",
249    "do",
250    "double",
251    "else",
252    "enum",
253    "explicit",
254    "export",
255    "extern",
256    "float",
257    "for",
258    "friend",
259    "goto",
260    "if",
261    "inline",
262    "int",
263    "long",
264    "namespace",
265    "new",
266    "operator",
267    "private",
268    "protected",
269    "public",
270    "register",
271    "return",
272    "short",
273    "signed",
274    "sizeof",
275    "static",
276    "struct",
277    "switch",
278    "template",
279    "this",
280    "throw",
281    "try",
282    "typedef",
283    "typename",
284    "union",
285    "unsigned",
286    "using",
287    "virtual",
288    "void",
289    "volatile",
290    "while",
291  ],
292  literals: &["NULL", "false", "nullptr", "true"],
293  fn_call: true,
294};
295const RUST: Profile = Profile {
296  line_comments: &["//"],
297  block_comment: Some(("/*", "*/")),
298  strings: &[DQ, SQ],
299  keywords: &[
300    "Self", "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "fn",
301    "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "static",
302    "struct", "super", "trait", "type", "unsafe", "use", "where", "while",
303  ],
304  literals: &["None", "false", "true"],
305  fn_call: true,
306};
307const JAVA: Profile = Profile {
308  line_comments: &["//"],
309  block_comment: Some(("/*", "*/")),
310  strings: &[DQ, SQ],
311  keywords: &[
312    "abstract",
313    "assert",
314    "boolean",
315    "break",
316    "byte",
317    "case",
318    "catch",
319    "char",
320    "class",
321    "const",
322    "continue",
323    "default",
324    "do",
325    "double",
326    "else",
327    "enum",
328    "extends",
329    "final",
330    "finally",
331    "float",
332    "for",
333    "goto",
334    "if",
335    "implements",
336    "import",
337    "instanceof",
338    "int",
339    "interface",
340    "long",
341    "native",
342    "new",
343    "package",
344    "private",
345    "protected",
346    "public",
347    "return",
348    "short",
349    "static",
350    "strictfp",
351    "super",
352    "switch",
353    "synchronized",
354    "this",
355    "throw",
356    "throws",
357    "transient",
358    "try",
359    "void",
360    "volatile",
361    "while",
362  ],
363  literals: &["false", "null", "true"],
364  fn_call: true,
365};
366const KOTLIN: Profile = Profile {
367  line_comments: &["//"],
368  block_comment: Some(("/*", "*/")),
369  strings: &[TRIPLE_DQ, DQ, SQ],
370  keywords: &[
371    "as",
372    "break",
373    "by",
374    "catch",
375    "class",
376    "constructor",
377    "continue",
378    "data",
379    "do",
380    "else",
381    "enum",
382    "finally",
383    "for",
384    "fun",
385    "if",
386    "import",
387    "in",
388    "interface",
389    "is",
390    "object",
391    "package",
392    "private",
393    "protected",
394    "public",
395    "return",
396    "sealed",
397    "super",
398    "this",
399    "throw",
400    "try",
401    "typealias",
402    "val",
403    "var",
404    "when",
405    "while",
406  ],
407  literals: &["false", "null", "true"],
408  fn_call: true,
409};
410const GO: Profile = Profile {
411  line_comments: &["//"],
412  block_comment: Some(("/*", "*/")),
413  strings: &[RAW, DQ, SQ],
414  keywords: &[
415    "break",
416    "case",
417    "chan",
418    "const",
419    "continue",
420    "default",
421    "defer",
422    "fallthrough",
423    "for",
424    "func",
425    "go",
426    "goto",
427    "if",
428    "import",
429    "interface",
430    "map",
431    "package",
432    "range",
433    "return",
434    "select",
435    "struct",
436    "switch",
437    "type",
438    "var",
439  ],
440  literals: &["false", "nil", "true"],
441  fn_call: true,
442};
443const PYTHON: Profile = Profile {
444  line_comments: &["#"],
445  block_comment: None,
446  strings: &[TRIPLE_DQ, TRIPLE_SQ, DQ, SQ],
447  keywords: &[
448    "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except",
449    "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise",
450    "return", "try", "while", "with", "yield",
451  ],
452  literals: &["False", "None", "True"],
453  fn_call: true,
454};
455const JS: Profile = Profile {
456  line_comments: &["//"],
457  block_comment: Some(("/*", "*/")),
458  strings: &[RAW, DQ, SQ],
459  keywords: &[
460    "async",
461    "await",
462    "break",
463    "case",
464    "catch",
465    "class",
466    "const",
467    "continue",
468    "debugger",
469    "default",
470    "delete",
471    "do",
472    "else",
473    "export",
474    "extends",
475    "finally",
476    "for",
477    "from",
478    "function",
479    "get",
480    "if",
481    "import",
482    "in",
483    "instanceof",
484    "let",
485    "new",
486    "of",
487    "return",
488    "set",
489    "static",
490    "super",
491    "switch",
492    "throw",
493    "try",
494    "typeof",
495    "var",
496    "void",
497    "while",
498    "with",
499    "yield",
500  ],
501  literals: &["false", "null", "true", "undefined"],
502  fn_call: true,
503};
504const RUBY: Profile = Profile {
505  line_comments: &["#"],
506  block_comment: None,
507  strings: &[DQ, SQ],
508  keywords: &[
509    "BEGIN", "END", "alias", "begin", "break", "case", "class", "def", "defined", "do", "else", "elsif", "end",
510    "ensure", "for", "if", "in", "module", "next", "redo", "rescue", "retry", "return", "self", "super", "then",
511    "undef", "unless", "until", "when", "while", "yield",
512  ],
513  literals: &["false", "nil", "true"],
514  fn_call: true,
515};
516const SHELL: Profile = Profile {
517  line_comments: &["#"],
518  block_comment: None,
519  strings: &[DQ, SQ],
520  keywords: &[
521    "case", "do", "done", "elif", "else", "esac", "fi", "for", "function", "if", "in", "select", "then", "time",
522    "until", "while",
523  ],
524  literals: &["false", "true"],
525  fn_call: true,
526};
527const SQL: Profile = Profile {
528  line_comments: &["--"],
529  block_comment: Some(("/*", "*/")),
530  strings: &[SQ, DQ],
531  keywords: &[
532    "ADD",
533    "ALTER",
534    "AND",
535    "AS",
536    "ASC",
537    "BEGIN",
538    "BETWEEN",
539    "BY",
540    "CASE",
541    "CREATE",
542    "DELETE",
543    "DESC",
544    "DISTINCT",
545    "DROP",
546    "ELSE",
547    "END",
548    "EXISTS",
549    "FROM",
550    "FULL",
551    "GROUP",
552    "HAVING",
553    "IN",
554    "INDEX",
555    "INNER",
556    "INSERT",
557    "INTO",
558    "IS",
559    "JOIN",
560    "LEFT",
561    "LIKE",
562    "LIMIT",
563    "NOT",
564    "ON",
565    "OR",
566    "ORDER",
567    "OUTER",
568    "PRIMARY",
569    "REFERENCES",
570    "RIGHT",
571    "SELECT",
572    "SET",
573    "TABLE",
574    "THEN",
575    "UNION",
576    "UNIQUE",
577    "UPDATE",
578    "VALUES",
579    "WHEN",
580    "WHERE",
581  ],
582  literals: &["FALSE", "NULL", "TRUE"],
583  fn_call: true,
584};
585const CSS: Profile = Profile {
586  line_comments: &[],
587  block_comment: Some(("/*", "*/")),
588  strings: &[DQ, SQ],
589  keywords: &[],
590  literals: &[],
591  fn_call: true,
592};
593const TOML: Profile = Profile {
594  line_comments: &["#"],
595  block_comment: None,
596  strings: &[TRIPLE_DQ, TRIPLE_SQ, DQ, SQ],
597  keywords: &[],
598  literals: &["false", "true"],
599  fn_call: false,
600};
601const YAML: Profile = Profile {
602  line_comments: &["#"],
603  block_comment: None,
604  strings: &[DQ, SQ],
605  keywords: &[],
606  literals: &["false", "null", "true", "~"],
607  fn_call: false,
608};
609const JSON: Profile = Profile {
610  line_comments: &[],
611  block_comment: None,
612  strings: &[DQ],
613  keywords: &[],
614  literals: &["false", "null", "true"],
615  fn_call: false,
616};
617
618fn profile_for(path: &str) -> Option<&'static Profile> {
619  let ext = path.rsplit_once('.')?.1.to_ascii_lowercase();
620  match ext.as_str() {
621    "rs" => Some(&RUST),
622    "c" | "cc" | "cpp" | "h" | "hpp" => Some(&C_LIKE),
623    "java" => Some(&JAVA),
624    "kt" | "kts" => Some(&KOTLIN),
625    "go" => Some(&GO),
626    "py" => Some(&PYTHON),
627    "js" | "jsx" | "mjs" | "ts" | "tsx" => Some(&JS),
628    "rb" => Some(&RUBY),
629    "bash" | "sh" | "zsh" => Some(&SHELL),
630    "sql" => Some(&SQL),
631    "css" => Some(&CSS),
632    "toml" => Some(&TOML),
633    "yaml" | "yml" => Some(&YAML),
634    "json" => Some(&JSON),
635    _ => None,
636  }
637}
638
639#[cfg(test)]
640mod tests {
641  use super::*;
642
643  fn one(path: &str, line: &str) -> String {
644    highlight_lines(path, &[line]).expect("test path has a profile").remove(0)
645  }
646
647  #[test]
648  fn profile_smoke_tests_cover_every_language_family() {
649    let cases: [(&str, &str, &[&str]); 14] = [
650      (
651        "x.rs",
652        "fn main() { call(1, \"x\"); true } // hi",
653        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
654      ),
655      (
656        "x.cpp",
657        "return call(0xff, \"x\", true); // hi",
658        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
659      ),
660      (
661        "x.java",
662        "class X { call(1, \"x\", true); } // hi",
663        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
664      ),
665      (
666        "x.kt",
667        "fun main() = call(1, \"x\", true) // hi",
668        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
669      ),
670      (
671        "x.go",
672        "func main() { call(1, \"x\", nil) } // hi",
673        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
674      ),
675      (
676        "x.py",
677        "def f(): return call(1, \"x\", None) # hi",
678        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
679      ),
680      (
681        "x.tsx",
682        "const x = call(1, \"x\", undefined); // hi",
683        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
684      ),
685      (
686        "x.rb",
687        "def f; call(1, \"x\", nil); end # hi",
688        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
689      ),
690      (
691        "x.sh",
692        "if call(1, \"x\", true); then echo ok; fi # hi",
693        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
694      ),
695      (
696        "x.sql",
697        "SELECT call(1, 'x', NULL) FROM t -- hi",
698        &["tok-kw", "tok-fn", "tok-num", "tok-str", "tok-lit", "tok-com"],
699      ),
700      ("x.css", "a { width: calc(12px); content: \"x\"; } /* hi */", &["tok-fn", "tok-num", "tok-str", "tok-com"]),
701      ("x.toml", "value = \"x\"; count = 1; enabled = true # hi", &["tok-str", "tok-num", "tok-lit", "tok-com"]),
702      ("x.yaml", "value: \"x\", count: 1, enabled: true # hi", &["tok-str", "tok-num", "tok-lit", "tok-com"]),
703      ("x.json", "{\"value\": 1, \"ok\": true}", &["tok-str", "tok-num", "tok-lit"]),
704    ];
705    for (path, source, expected) in cases {
706      let html = one(path, source);
707      for class in expected {
708        assert!(html.contains(class), "{path} should contain {class}: {html}");
709      }
710    }
711  }
712
713  #[test]
714  fn profile_word_lists_stay_sorted_for_binary_search() {
715    for profile in [&RUST, &C_LIKE, &JAVA, &KOTLIN, &GO, &PYTHON, &JS, &RUBY, &SHELL, &SQL, &CSS, &TOML, &YAML, &JSON] {
716      assert!(profile.keywords.windows(2).all(|pair| pair[0] <= pair[1]));
717      assert!(profile.literals.windows(2).all(|pair| pair[0] <= pair[1]));
718    }
719  }
720
721  #[test]
722  fn state_carries_across_multiline_constructs() {
723    let rust = highlight_lines("x.rs", &["/* open", "still */ let x = 1;"]).expect("known profile");
724    assert!(rust[1].starts_with("<span class=\"tok-com\">still */</span> <span class=\"tok-kw\">let</span>"));
725    let python = highlight_lines("x.py", &["\"\"\"open", "still", "\"\"\" + 2"]).expect("known profile");
726    assert!(python[1].starts_with("<span class=\"tok-str\">still</span>"));
727    assert!(python[2].starts_with("<span class=\"tok-str\">&quot;&quot;&quot;</span>"));
728  }
729
730  #[test]
731  fn hunk_keeps_old_and_new_state_separate() {
732    let lines = vec![
733      Line::Del { old: 1, text: "/* old opens".into() },
734      Line::Add { new: 1, text: "new is plain */".into() },
735      Line::Del { old: 2, text: "old closes */".into() },
736      Line::Add { new: 2, text: "let value = true;".into() },
737    ];
738    let html = highlight_hunk("x.rs", &lines).expect("known profile");
739    assert!(html[2].starts_with("<span class=\"tok-com\">"));
740    assert!(html[3].starts_with("<span class=\"tok-kw\">let</span>"));
741  }
742
743  #[test]
744  fn unknown_and_deliberately_plain_paths_fall_back() {
745    assert_eq!(highlight_lines("README.md", &["# heading"]), None);
746    assert_eq!(highlight_lines("page.html", &["<b>x</b>"]), None);
747  }
748
749  #[test]
750  fn hostile_input_is_escaped_and_spans_preserve_exact_escaped_text() {
751    let source = "const x = \"</span><script>&'\\\"\"; // <evil>";
752    let html = one("x.js", source);
753    assert!(!html.contains("<script>"));
754    let stripped = html
755      .replace("<span class=\"tok-com\">", "")
756      .replace("<span class=\"tok-str\">", "")
757      .replace("<span class=\"tok-num\">", "")
758      .replace("<span class=\"tok-kw\">", "")
759      .replace("<span class=\"tok-lit\">", "")
760      .replace("<span class=\"tok-fn\">", "")
761      .replace("</span>", "");
762    let mut escaped = String::new();
763    escape_into(&mut escaped, source);
764    assert_eq!(stripped, escaped);
765  }
766}
packdiff_dto/highlight.rs

packdiff_dto/
highlight.rs
