use super::code_highlight_node::create_code_highlight_node;
use super::flat_structure_utils::plainify_code_content;
use crate::node::NodeRef;
use crate::nodes::line_break::create_line_break_node;
use crate::nodes::tab::create_tab_node;
use crate::selection_insert_nodes::{RawTextVisitor, tokenize_raw_text};
#[derive(Clone, Debug, PartialEq)]
pub enum CodeToken {
Text(String),
Token(Token),
}
impl CodeToken {
pub fn token(token_type: &str, text: &str) -> CodeToken {
CodeToken::Token(Token {
token_type: token_type.to_owned(),
alias: None,
content: TokenContent::Text(text.to_owned()),
})
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Token {
pub token_type: String,
pub alias: Option<String>,
pub content: TokenContent,
}
#[derive(Clone, Debug, PartialEq)]
pub enum TokenContent {
Text(String),
Token(Box<Token>),
Tokens(Vec<CodeToken>),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CodeLanguageState {
Loaded,
Loading,
Unsupported,
}
pub trait Tokenizer {
fn default_language(&self) -> Option<&str>;
fn language_state(&self, language: &str) -> CodeLanguageState;
fn tokenize(&self, code: &str, language: &str) -> Vec<CodeToken>;
fn load_language(&self, language: &str, on_ready: Box<dyn FnOnce()>) {
let _ = (language, on_ready);
}
fn tokenize_code_node(&self, code_node: &NodeRef, language: Option<&str>) -> Vec<NodeRef> {
let code = code_node.get_text_content();
match language
.filter(|l| !l.is_empty())
.or(self.default_language())
{
None => plainify_code_content(&code),
Some(language) => {
map_tokens_to_lexical_structure(&self.tokenize(&code, language), None)
}
}
}
}
struct MapTokens<'a> {
nodes: &'a mut Vec<NodeRef>,
highlight_type: Option<&'a str>,
}
impl RawTextVisitor for MapTokens<'_> {
fn linebreak(&mut self) {
self.nodes.push(create_line_break_node());
}
fn tab(&mut self) {
self.nodes.push(create_tab_node());
}
fn text(&mut self, text: &str) {
self.nodes
.push(create_code_highlight_node(text, self.highlight_type));
}
}
pub fn map_tokens_to_lexical_structure(
tokens: &[CodeToken],
highlight_type: Option<&str>,
) -> Vec<NodeRef> {
let mut nodes = Vec::new();
add_tokens(tokens, highlight_type, &mut nodes);
nodes
}
fn add_tokens(tokens: &[CodeToken], highlight_type: Option<&str>, nodes: &mut Vec<NodeRef>) {
for token in tokens {
match token {
CodeToken::Text(text) => tokenize_raw_text(
text,
&mut MapTokens {
nodes,
highlight_type,
},
),
CodeToken::Token(token) => match &token.content {
TokenContent::Text(text) => {
let highlight_type = match (&token.alias, token.token_type.as_str()) {
(Some(alias), "prefix") => alias.as_str(),
_ => token.token_type.as_str(),
};
tokenize_raw_text(
text,
&mut MapTokens {
nodes,
highlight_type: Some(highlight_type),
},
);
}
TokenContent::Tokens(content) => add_tokens(
content,
(token.token_type != "unchanged").then_some(token.token_type.as_str()),
nodes,
),
TokenContent::Token(_) => {}
},
}
}
}
#[derive(Clone, Debug, Default)]
pub struct PlainTokenizer {
pub default_language: Option<String>,
}
impl Tokenizer for PlainTokenizer {
fn default_language(&self) -> Option<&str> {
self.default_language.as_deref()
}
fn language_state(&self, _language: &str) -> CodeLanguageState {
CodeLanguageState::Unsupported
}
fn tokenize(&self, code: &str, _language: &str) -> Vec<CodeToken> {
vec![CodeToken::Text(code.to_owned())]
}
}