wdl_format/token/
pre.rs

1//! Tokens emitted during the formatting of particular elements.
2
3use std::collections::HashSet;
4use std::rc::Rc;
5
6use wdl_ast::DOC_COMMENT_PREFIX;
7use wdl_ast::Directive;
8use wdl_ast::SyntaxKind;
9use wdl_ast::SyntaxTokenExt;
10
11use crate::Comment;
12use crate::Token;
13use crate::TokenStream;
14use crate::Trivia;
15use crate::TriviaBlankLineSpacingPolicy;
16
17/// A token that can be written by elements.
18///
19/// These are tokens that are intended to be written directly by elements to a
20/// [`TokenStream`](super::TokenStream) consisting of [`PreToken`]s. Note that
21/// this will transformed into a [`TokenStream`](super::TokenStream) of
22/// [`PostToken`](super::PostToken)s by a
23/// [`Postprocessor`](super::Postprocessor) (authors of elements are never
24/// expected to write [`PostToken`](super::PostToken)s directly).
25#[derive(Clone, Debug, Eq, PartialEq)]
26pub enum PreToken {
27    /// A non-trivial blank line.
28    ///
29    /// This will not be ignored by the postprocessor (unlike
30    /// [`Trivia::BlankLine`] which is potentially ignored).
31    BlankLine,
32
33    /// The end of a line.
34    LineEnd,
35
36    /// The end of a word.
37    WordEnd,
38
39    /// The start of an indented block.
40    IndentStart,
41
42    /// The end of an indented block.
43    IndentEnd,
44
45    /// How to handle trivial blank lines from this point onwards.
46    LineSpacingPolicy(TriviaBlankLineSpacingPolicy),
47
48    /// Literal text.
49    Literal(Rc<String>, SyntaxKind),
50
51    /// Trivia.
52    Trivia(Trivia),
53
54    /// A temporary indent start. Used in command section formatting.
55    ///
56    /// Command sections must account for indentation from both the
57    /// WDL context and the embedded Bash context, so this is used to
58    /// add additional indentation from the Bash context.
59    TempIndentStart(Rc<String>),
60
61    /// A temporary indent end. Used in command section formatting.
62    ///
63    /// See [`PreToken::TempIndentStart`] for more information.
64    TempIndentEnd,
65}
66
67impl std::fmt::Display for PreToken {
68    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
69        match self {
70            PreToken::BlankLine => write!(f, "<BlankLine>"),
71            PreToken::LineEnd => write!(f, "<EndOfLine>"),
72            PreToken::WordEnd => write!(f, "<WordEnd>"),
73            PreToken::IndentStart => write!(f, "<IndentStart>"),
74            PreToken::IndentEnd => write!(f, "<IndentEnd>"),
75            PreToken::LineSpacingPolicy(policy) => {
76                write!(f, "<LineSpacingPolicy@{policy:?}>")
77            }
78            PreToken::Literal(value, kind) => {
79                write!(f, "<Literal-{kind:?}@{value}>")
80            }
81            PreToken::Trivia(trivia) => match trivia {
82                Trivia::BlankLine => {
83                    write!(f, "<OptionalBlankLine>")
84                }
85                Trivia::Comment(comment) => match comment {
86                    Comment::Directive(directive) => {
87                        write!(f, "<Comment-Directive@{directive:?}>")
88                    }
89                    Comment::Documentation(documentation) => {
90                        write!(f, "<Comment-Documentation@{documentation}>")
91                    }
92                    Comment::Preceding(value) => {
93                        write!(f, "<Comment-Preceding@{value}>")
94                    }
95                    Comment::Inline(value) => {
96                        write!(f, "<Comment-Inline@{value}>")
97                    }
98                },
99            },
100            PreToken::TempIndentStart(value) => write!(f, "<TempIndentStart@{value}>"),
101            PreToken::TempIndentEnd => write!(f, "<TempIndentEnd>"),
102        }
103    }
104}
105
106impl Token for PreToken {
107    /// Returns a displayable version of the token.
108    fn display<'a>(&'a self, _config: &'a crate::Config) -> impl std::fmt::Display {
109        self
110    }
111}
112
113impl TokenStream<PreToken> {
114    /// Inserts a blank line token to the stream if the stream does not already
115    /// end with a blank line. This will replace any [`Trivia::BlankLine`]
116    /// tokens with [`PreToken::BlankLine`].
117    pub fn blank_line(&mut self) {
118        self.trim_while(|t| matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine)));
119        self.0.push(PreToken::BlankLine);
120    }
121
122    /// Inserts an end of line token to the stream if the stream does not
123    /// already end with an end of line token.
124    ///
125    /// This will also trim any trailing [`PreToken::WordEnd`] tokens.
126    pub fn end_line(&mut self) {
127        self.trim_while(|t| matches!(t, PreToken::WordEnd | PreToken::LineEnd));
128        self.0.push(PreToken::LineEnd);
129    }
130
131    /// Inserts a word end token to the stream if the stream does not already
132    /// end with a word end token.
133    pub fn end_word(&mut self) {
134        self.trim_end(&PreToken::WordEnd);
135        self.0.push(PreToken::WordEnd);
136    }
137
138    /// Inserts an indent start token to the stream. This will also end the
139    /// current line.
140    pub fn increment_indent(&mut self) {
141        self.end_line();
142        self.0.push(PreToken::IndentStart);
143    }
144
145    /// Inserts an indent end token to the stream. This will also end the
146    /// current line.
147    pub fn decrement_indent(&mut self) {
148        self.end_line();
149        self.0.push(PreToken::IndentEnd);
150    }
151
152    /// Inserts a trivial blank lines "always allowed" context change.
153    pub fn allow_blank_lines(&mut self) {
154        self.0.push(PreToken::LineSpacingPolicy(
155            TriviaBlankLineSpacingPolicy::Always,
156        ));
157    }
158
159    /// Inserts a trivial blank lines "not allowed after comments" context
160    /// change.
161    pub fn ignore_trailing_blank_lines(&mut self) {
162        self.0.push(PreToken::LineSpacingPolicy(
163            TriviaBlankLineSpacingPolicy::RemoveTrailingBlanks,
164        ));
165    }
166
167    /// Inserts any preceding trivia into the stream.
168    ///
169    /// This will consolidate all doc comments and directive comments which
170    /// precede this token.
171    ///
172    /// # Panics
173    ///
174    /// This will panic if the provided token is itself trivia, as trivia
175    /// cannot have trivia.
176    fn push_preceding_trivia(&mut self, token: &wdl_ast::Token) {
177        assert!(!token.inner().kind().is_trivia());
178        let preceding_trivia = token.inner().preceding_trivia();
179        let mut documentation = String::new();
180        let mut trivia = Vec::new();
181        let mut exceptions = HashSet::new();
182        for token in preceding_trivia {
183            match token.kind() {
184                SyntaxKind::Whitespace => {
185                    if !self.0.last().is_some_and(|t| {
186                        matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine))
187                    }) {
188                        trivia.push(PreToken::Trivia(Trivia::BlankLine));
189                    }
190                }
191                SyntaxKind::Comment => {
192                    if let Some(t) = token.text().strip_prefix(DOC_COMMENT_PREFIX) {
193                        // do not `trim()` the token as the whitespace may
194                        // have syntactical meaning in markdown
195                        documentation.push_str(t);
196                        documentation.push('\n');
197                    } else if let Ok(directive) = token.text().parse::<Directive>() {
198                        match directive {
199                            Directive::Except(e) => exceptions.extend(e),
200                        }
201                    } else {
202                        let comment = PreToken::Trivia(Trivia::Comment(Comment::Preceding(
203                            Rc::new(token.text().trim_end().to_string()),
204                        )));
205                        trivia.push(comment);
206                    }
207                }
208                _ => unreachable!("unexpected trivia: {:?}", token),
209            };
210        }
211
212        let mut trivia = trivia.into_iter().peekable();
213        // Preserve any leading blank lines
214        if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
215            self.0.push(trivia.next().unwrap());
216        }
217        let mut docs_present = false;
218        if !documentation.is_empty() {
219            docs_present = true;
220            let comment = PreToken::Trivia(Trivia::Comment(Comment::Documentation(Rc::new(
221                documentation,
222            ))));
223            self.0.push(comment);
224
225            // don't allow documentation to "float" above the item being documented
226            if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
227                let _ = trivia.next();
228            }
229        }
230        for token in trivia {
231            self.0.push(token);
232        }
233        if docs_present && let Some(PreToken::Trivia(Trivia::BlankLine)) = self.0.last() {
234            // don't allow documentation to "float" above the item being documented
235            self.0.pop();
236        }
237        if !exceptions.is_empty() {
238            let comment = PreToken::Trivia(Trivia::Comment(Comment::Directive(Rc::new(
239                Directive::Except(exceptions),
240            ))));
241            self.0.push(comment);
242        }
243    }
244
245    /// Inserts any inline trivia into the stream.
246    ///
247    /// # Panics
248    ///
249    /// This will panic if the provided token is itself trivia, as trivia
250    /// cannot have trivia.
251    fn push_inline_trivia(&mut self, token: &wdl_ast::Token) {
252        assert!(!token.inner().kind().is_trivia());
253        if let Some(token) = token.inner().inline_comment() {
254            let inline_comment = PreToken::Trivia(Trivia::Comment(Comment::Inline(Rc::new(
255                token.text().trim_end().to_owned(),
256            ))));
257            self.0.push(inline_comment);
258        }
259    }
260
261    /// Pushes an AST token into the stream.
262    ///
263    /// This will also push any preceding or inline trivia into the stream.
264    /// Any token may have preceding or inline trivia, unless that token is
265    /// itself trivia (i.e. trivia cannot have trivia).
266    ///
267    /// # Panics
268    ///
269    /// This will panic if the provided token is trivia.
270    pub fn push_ast_token(&mut self, token: &wdl_ast::Token) {
271        self.push_preceding_trivia(token);
272        self.0.push(PreToken::Literal(
273            Rc::new(token.inner().text().to_owned()),
274            token.inner().kind(),
275        ));
276        self.push_inline_trivia(token);
277    }
278
279    /// Pushes a literal string into the stream in place of an AST token.
280    ///
281    /// This will insert any trivia that would have been inserted with the AST
282    /// token.
283    ///
284    /// # Panics
285    ///
286    /// This will panic if the provided token is trivia.
287    pub fn push_literal_in_place_of_token(&mut self, token: &wdl_ast::Token, replacement: String) {
288        self.push_preceding_trivia(token);
289        self.0.push(PreToken::Literal(
290            Rc::new(replacement),
291            token.inner().kind(),
292        ));
293        self.push_inline_trivia(token);
294    }
295
296    /// Pushes a literal string into the stream.
297    ///
298    /// This will not insert any trivia.
299    pub fn push_literal(&mut self, value: String, kind: SyntaxKind) {
300        self.0.push(PreToken::Literal(Rc::new(value), kind));
301    }
302}
wdl_format/token/pre.rs

wdl_format/token/
pre.rs
