termimad 0.14.3

Markdown Renderer for the Terminal
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

use {
    crate::*,
    minimad::*,
    unicode_width::UnicodeWidthChar,
};

#[derive(Debug)]
pub(crate) struct Token<'s> {
    pub compound: Compound<'s>,
    pub blank: bool,
    pub width: usize,
    pub start_in_compound: usize,
    pub end_in_compound: usize,
}

impl<'s> Token<'s> {
    pub fn to_compound(&self) -> Compound<'s> {
        let mut compound = self.compound.clone();
        compound.set_str(&self.compound.src[self.start_in_compound..self.end_in_compound]);
        compound
    }
}

/// Cut a composite into token, each one being either only spaces or without space, and
/// each one from one compound
pub(crate) fn tokenize<'s, 'c>(
    composite: &'c Composite<'s>,
    max_token_width: usize,
) -> Vec<Token<'s>> {
    let mut tokens: Vec<Token<'s>> = Vec::new();
    for compound in &composite.compounds {
        let mut token: Option<Token> = None;
        for (idx, char) in compound.src.char_indices() {
            let blank = char.is_whitespace() && !compound.code;
            let char_width = char.width().unwrap_or(0);
            if let Some(token) = token.as_mut() {
                if token.blank == blank && token.width + char_width <= max_token_width {
                    token.width += char_width;
                    token.end_in_compound += char.len_utf8();
                    continue;
                }
            }
            let new_token = Token {
                compound: compound.clone(),
                blank,
                width: char_width,
                start_in_compound: idx,
                end_in_compound: idx + char.len_utf8(),
            };
            if let Some(token) = token.replace(new_token) {
                tokens.push(token);
            }
        }
        if let Some(token) = token {
            tokens.push(token);
        }
    }
    tokens
}