cargo_cargofmt/toml/
tokens.rs

1use std::borrow::Cow;
2
3pub use toml_parser::decoder::Encoding;
4pub use toml_parser::decoder::ScalarKind;
5pub use toml_parser::parser::EventKind as TokenKind;
6
7#[derive(Debug)]
8pub struct TomlTokens<'i> {
9    pub tokens: Vec<TomlToken<'i>>,
10    input_len: usize,
11}
12
13impl<'i> TomlTokens<'i> {
14    pub fn parse(input: &'i str) -> Self {
15        let source = toml_parser::Source::new(input);
16        let tokens = source.lex().into_vec();
17
18        let mut events = Vec::with_capacity(tokens.len());
19        toml_parser::parser::parse_document(&tokens, &mut events, &mut ());
20
21        let tokens = events
22            .into_iter()
23            .map(|e| {
24                let raw = source.get(e).expect("already validated");
25                let mut decoded = None;
26                let mut scalar = None;
27                match e.kind() {
28                    TokenKind::SimpleKey => {
29                        let mut d = Cow::Borrowed("");
30                        raw.decode_key(&mut d, &mut ());
31                        decoded = Some(d);
32                    }
33                    TokenKind::Scalar => {
34                        let mut d = Cow::Borrowed("");
35                        let s = raw.decode_scalar(&mut d, &mut ());
36                        if matches!(s, ScalarKind::String) {
37                            decoded = Some(d);
38                        } else {
39                            scalar = Some(s);
40                        }
41                    }
42                    _ => {}
43                }
44                TomlToken {
45                    kind: e.kind(),
46                    encoding: e.encoding(),
47                    decoded,
48                    scalar,
49                    raw: Cow::Borrowed(raw.as_str()),
50                }
51            })
52            .collect();
53
54        Self {
55            tokens,
56            input_len: input.len(),
57        }
58    }
59
60    pub fn indices(&self) -> impl Iterator<Item = usize> {
61        0..self.tokens.len()
62    }
63
64    pub fn is_empty(&self) -> bool {
65        self.tokens.is_empty()
66    }
67
68    pub fn len(&self) -> usize {
69        self.tokens.len()
70    }
71
72    pub fn trim_empty_whitespace(&mut self) {
73        self.tokens
74            .retain(|t| !(matches!(t.kind, TokenKind::Whitespace) && t.raw.is_empty()));
75    }
76
77    #[allow(clippy::inherent_to_string_shadow_display)]
78    pub fn to_string(&self) -> String {
79        use std::fmt::Write as _;
80
81        let mut result = String::with_capacity(self.input_len);
82        write!(&mut result, "{self}").unwrap();
83        result
84    }
85}
86
87impl std::fmt::Display for TomlTokens<'_> {
88    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
89        for token in &self.tokens {
90            token.fmt(f)?;
91        }
92        Ok(())
93    }
94}
95
96#[derive(Debug)]
97pub struct TomlToken<'i> {
98    pub kind: TokenKind,
99    pub encoding: Option<Encoding>,
100    pub decoded: Option<Cow<'i, str>>,
101    pub scalar: Option<ScalarKind>,
102    pub raw: Cow<'i, str>,
103}
104
105impl TomlToken<'_> {
106    pub const EMPTY: Self = Self {
107        kind: TokenKind::Whitespace,
108        encoding: None,
109        decoded: None,
110        scalar: None,
111        raw: Cow::Borrowed(""),
112    };
113    pub const SPACE: Self = Self {
114        kind: TokenKind::Whitespace,
115        encoding: None,
116        decoded: None,
117        scalar: None,
118        raw: Cow::Borrowed(" "),
119    };
120    pub const NL: Self = Self {
121        kind: TokenKind::Newline,
122        encoding: None,
123        decoded: None,
124        scalar: None,
125        raw: Cow::Borrowed("\n"), // assuming operating on normalized newlines
126    };
127    pub const VAL_SEP: Self = Self {
128        kind: TokenKind::ValueSep,
129        encoding: None,
130        decoded: None,
131        scalar: None,
132        raw: Cow::Borrowed(","),
133    };
134}
135
136impl std::fmt::Display for TomlToken<'_> {
137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138        self.raw.fmt(f)
139    }
140}
cargo_cargofmt/toml/tokens.rs

cargo_cargofmt/toml/
tokens.rs
