mod cache;
mod language;
mod languages;
mod lines;
#[cfg(test)]
mod lines_tests;
mod registry;
#[cfg(test)]
mod registry_tests;
mod scan;
#[cfg(test)]
mod tests;
use std::ops::Range;
pub use language::{BlockComment, CodeLanguage, CodeMode, KeyRule, NumberForms, StringForm};
pub(crate) use lines::LineCache;
pub use lines::{BufferId, CodeEdit, CodePoint, LineState};
pub use registry::{
CodeTokenizer, Tokenizer, TokenizerHandle, knows, languages, register_tokenizer, register_tokenizer_with_aliases,
remove_tokenizer, tokenizer, tokens,
};
pub use scan::ScanState;
pub type TokenName = crate::app::theme::RoleName;
pub mod token {
pub const KEYWORD: &str = "keyword";
pub const KEYWORD_CONTROL: &str = "keyword.control";
pub const STRING: &str = "string";
pub const STRING_ESCAPE: &str = "string.escape";
pub const NUMBER: &str = "number";
pub const COMMENT: &str = "comment";
pub const COMMENT_DOC: &str = "comment.doc";
pub const TYPE: &str = "type";
pub const TYPE_BUILTIN: &str = "type.builtin";
pub const FUNCTION: &str = "function";
pub const FUNCTION_MACRO: &str = "function.macro";
pub const VARIABLE: &str = "variable";
pub const VARIABLE_BUILTIN: &str = "variable.builtin";
pub const CONSTANT: &str = "constant";
pub const CONSTANT_LANGUAGE: &str = "constant.language";
pub const PROPERTY: &str = "property";
pub const ATTRIBUTE: &str = "attribute";
pub const OPERATOR: &str = "operator";
pub const PUNCTUATION: &str = "punctuation";
pub const TAG: &str = "tag";
pub const REGEX: &str = "regex";
pub const NAMESPACE: &str = "namespace";
pub const LABEL: &str = "label";
pub const MARKUP_HEADING: &str = "markup.heading";
pub const MARKUP_BOLD: &str = "markup.bold";
pub const MARKUP_ITALIC: &str = "markup.italic";
pub const MARKUP_CODE: &str = "markup.code";
pub const MARKUP_LINK: &str = "markup.link";
pub const MARKUP_LIST: &str = "markup.list";
pub const MARKUP_QUOTE: &str = "markup.quote";
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct CodeToken {
pub range: Range<usize>,
pub kind: TokenName,
}
impl CodeToken {
pub fn new(range: Range<usize>, kind: impl Into<TokenName>) -> Self {
Self {
range,
kind: kind.into(),
}
}
}
pub fn lookup_names(name: &str) -> impl Iterator<Item = &str> {
let mut next = Some(name);
std::iter::from_fn(move || {
let name = next?;
next = name.rfind('.').map(|dot| &name[..dot]).filter(|rest| !rest.is_empty());
Some(name)
})
}
pub(crate) fn normalized(code: &str, mut tokens: Vec<CodeToken>) -> Vec<CodeToken> {
tokens.retain(|token| {
token.range.start < token.range.end
&& token.range.end <= code.len()
&& code.is_char_boundary(token.range.start)
&& code.is_char_boundary(token.range.end)
});
if !tokens.is_sorted_by_key(|token| token.range.start) {
tokens.sort_by_key(|token| token.range.start);
}
let mut end = 0;
tokens.retain(|token| {
let keep = token.range.start >= end;
if keep {
end = token.range.end;
}
keep
});
tokens
}
pub fn tokens_by_line(code: &str, tokens: &[CodeToken]) -> Vec<Vec<CodeToken>> {
let mut lines = Vec::new();
let mut next = 0;
let mut start = 0;
for line in code.split('\n') {
let end = start + line.len();
let mut own = Vec::new();
while next < tokens.len() && tokens[next].range.start < end + 1 {
let token = &tokens[next];
let from = token.range.start.max(start);
let to = token.range.end.min(end);
if from < to {
own.push(CodeToken {
range: from - start..to - start,
kind: token.kind,
});
}
if token.range.end > end + 1 {
break;
}
next += 1;
}
lines.push(own);
start = end + 1;
}
lines
}