wdl_format/token/
pre.rs

1//! Tokens emitted during the formatting of particular elements.
2
3use std::collections::HashSet;
4use std::rc::Rc;
5
6use wdl_ast::DOC_COMMENT_PREFIX;
7use wdl_ast::Directive;
8use wdl_ast::SyntaxKind;
9use wdl_ast::SyntaxTokenExt;
10
11use crate::Comment;
12use crate::NEWLINE;
13use crate::Token;
14use crate::TokenStream;
15use crate::Trivia;
16use crate::TriviaBlankLineSpacingPolicy;
17
18/// A token that can be written by elements.
19///
20/// These are tokens that are intended to be written directly by elements to a
21/// [`TokenStream`](super::TokenStream) consisting of [`PreToken`]s. Note that
22/// this will transformed into a [`TokenStream`](super::TokenStream) of
23/// [`PostToken`](super::PostToken)s by a
24/// [`Postprocessor`](super::Postprocessor) (authors of elements are never
25/// expected to write [`PostToken`](super::PostToken)s directly).
26#[derive(Clone, Debug, Eq, PartialEq)]
27pub enum PreToken {
28    /// A non-trivial blank line.
29    ///
30    /// This will not be ignored by the postprocessor (unlike
31    /// [`Trivia::BlankLine`] which is potentially ignored).
32    BlankLine,
33
34    /// The end of a line.
35    LineEnd,
36
37    /// The end of a word.
38    WordEnd,
39
40    /// The start of an indented block.
41    IndentStart,
42
43    /// The end of an indented block.
44    IndentEnd,
45
46    /// How to handle trivial blank lines from this point onwards.
47    LineSpacingPolicy(TriviaBlankLineSpacingPolicy),
48
49    /// Literal text.
50    Literal(Rc<String>, SyntaxKind),
51
52    /// Trivia.
53    Trivia(Trivia),
54
55    /// A temporary indent start. Used in command section formatting.
56    ///
57    /// Command sections must account for indentation from both the
58    /// WDL context and the embedded Bash context, so this is used to
59    /// add additional indentation from the Bash context.
60    TempIndentStart(Rc<String>),
61
62    /// A temporary indent end. Used in command section formatting.
63    ///
64    /// See [`PreToken::TempIndentStart`] for more information.
65    TempIndentEnd,
66}
67
68impl std::fmt::Display for PreToken {
69    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
70        match self {
71            PreToken::BlankLine => write!(f, "<BlankLine>"),
72            PreToken::LineEnd => write!(f, "<EndOfLine>"),
73            PreToken::WordEnd => write!(f, "<WordEnd>"),
74            PreToken::IndentStart => write!(f, "<IndentStart>"),
75            PreToken::IndentEnd => write!(f, "<IndentEnd>"),
76            PreToken::LineSpacingPolicy(policy) => {
77                write!(f, "<LineSpacingPolicy@{policy:?}>")
78            }
79            PreToken::Literal(value, kind) => {
80                write!(f, "<Literal-{kind:?}@{value}>")
81            }
82            PreToken::Trivia(trivia) => match trivia {
83                Trivia::BlankLine => {
84                    write!(f, "<OptionalBlankLine>")
85                }
86                Trivia::Comment(comment) => match comment {
87                    Comment::Directive(directive) => {
88                        write!(f, "<Comment-Directive@{directive:?}>")
89                    }
90                    Comment::Documentation(documentation) => {
91                        write!(f, "<Comment-Documentation@{documentation}>")
92                    }
93                    Comment::Preceding(value) => {
94                        write!(f, "<Comment-Preceding@{value}>")
95                    }
96                    Comment::Inline(value) => {
97                        write!(f, "<Comment-Inline@{value}>")
98                    }
99                },
100            },
101            PreToken::TempIndentStart(value) => write!(f, "<TempIndentStart@{value}>"),
102            PreToken::TempIndentEnd => write!(f, "<TempIndentEnd>"),
103        }
104    }
105}
106
107impl Token for PreToken {
108    /// Returns a displayable version of the token.
109    fn display<'a>(&'a self, _config: &'a crate::Config) -> impl std::fmt::Display {
110        self
111    }
112}
113
114impl TokenStream<PreToken> {
115    /// Inserts a blank line token to the stream if the stream does not already
116    /// end with a blank line. This will replace any [`Trivia::BlankLine`]
117    /// tokens with [`PreToken::BlankLine`].
118    pub fn blank_line(&mut self) {
119        self.trim_while(|t| matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine)));
120        self.0.push(PreToken::BlankLine);
121    }
122
123    /// Inserts an end of line token to the stream if the stream does not
124    /// already end with an end of line token.
125    ///
126    /// This will also trim any trailing [`PreToken::WordEnd`] tokens.
127    pub fn end_line(&mut self) {
128        self.trim_while(|t| matches!(t, PreToken::WordEnd | PreToken::LineEnd));
129        self.0.push(PreToken::LineEnd);
130    }
131
132    /// Inserts a word end token to the stream if the stream does not already
133    /// end with a word end token.
134    pub fn end_word(&mut self) {
135        self.trim_end(&PreToken::WordEnd);
136        self.0.push(PreToken::WordEnd);
137    }
138
139    /// Inserts an indent start token to the stream. This will also end the
140    /// current line.
141    pub fn increment_indent(&mut self) {
142        self.end_line();
143        self.0.push(PreToken::IndentStart);
144    }
145
146    /// Inserts an indent end token to the stream. This will also end the
147    /// current line.
148    pub fn decrement_indent(&mut self) {
149        self.end_line();
150        self.0.push(PreToken::IndentEnd);
151    }
152
153    /// Inserts a trivial blank lines "always allowed" context change.
154    pub fn allow_blank_lines(&mut self) {
155        self.0.push(PreToken::LineSpacingPolicy(
156            TriviaBlankLineSpacingPolicy::Always,
157        ));
158    }
159
160    /// Inserts a trivial blank lines "not allowed after comments" context
161    /// change.
162    pub fn ignore_trailing_blank_lines(&mut self) {
163        self.0.push(PreToken::LineSpacingPolicy(
164            TriviaBlankLineSpacingPolicy::RemoveTrailingBlanks,
165        ));
166    }
167
168    /// Inserts any preceding trivia into the stream.
169    ///
170    /// This will consolidate all doc comments and directive comments which
171    /// precede this token.
172    ///
173    /// # Panics
174    ///
175    /// This will panic if the provided token is itself trivia, as trivia
176    /// cannot have trivia.
177    fn push_preceding_trivia(&mut self, token: &wdl_ast::Token) {
178        assert!(!token.inner().kind().is_trivia());
179        let preceding_trivia = token.inner().preceding_trivia();
180        let mut documentation = String::new();
181        let mut trivia = Vec::new();
182        let mut exceptions = HashSet::new();
183        for token in preceding_trivia {
184            match token.kind() {
185                SyntaxKind::Whitespace => {
186                    if !self.0.last().is_some_and(|t| {
187                        matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine))
188                    }) {
189                        trivia.push(PreToken::Trivia(Trivia::BlankLine));
190                    }
191                }
192                SyntaxKind::Comment => {
193                    if let Some(t) = token.text().strip_prefix(DOC_COMMENT_PREFIX) {
194                        // do not `trim()` the token as the whitespace may
195                        // have syntactical meaning in markdown
196                        documentation.push_str(t);
197                        documentation.push_str(NEWLINE);
198                    } else if let Ok(directive) = token.text().parse::<Directive>() {
199                        match directive {
200                            Directive::Except(e) => exceptions.extend(e),
201                        }
202                    } else {
203                        let comment = PreToken::Trivia(Trivia::Comment(Comment::Preceding(
204                            Rc::new(token.text().trim_end().to_string()),
205                        )));
206                        trivia.push(comment);
207                    }
208                }
209                _ => unreachable!("unexpected trivia: {:?}", token),
210            };
211        }
212
213        let mut trivia = trivia.into_iter().peekable();
214        // Preserve any leading blank lines
215        if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
216            self.0.push(trivia.next().unwrap());
217        }
218        let mut docs_present = false;
219        if !documentation.is_empty() {
220            docs_present = true;
221            let comment = PreToken::Trivia(Trivia::Comment(Comment::Documentation(Rc::new(
222                documentation,
223            ))));
224            self.0.push(comment);
225
226            // don't allow documentation to "float" above the item being documented
227            if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
228                let _ = trivia.next();
229            }
230        }
231        for token in trivia {
232            self.0.push(token);
233        }
234        if docs_present && let Some(PreToken::Trivia(Trivia::BlankLine)) = self.0.last() {
235            // don't allow documentation to "float" above the item being documented
236            self.0.pop();
237        }
238        if !exceptions.is_empty() {
239            let comment = PreToken::Trivia(Trivia::Comment(Comment::Directive(Rc::new(
240                Directive::Except(exceptions),
241            ))));
242            self.0.push(comment);
243        }
244    }
245
246    /// Inserts any inline trivia into the stream.
247    ///
248    /// # Panics
249    ///
250    /// This will panic if the provided token is itself trivia, as trivia
251    /// cannot have trivia.
252    fn push_inline_trivia(&mut self, token: &wdl_ast::Token) {
253        assert!(!token.inner().kind().is_trivia());
254        if let Some(token) = token.inner().inline_comment() {
255            let inline_comment = PreToken::Trivia(Trivia::Comment(Comment::Inline(Rc::new(
256                token.text().trim_end().to_owned(),
257            ))));
258            self.0.push(inline_comment);
259        }
260    }
261
262    /// Pushes an AST token into the stream.
263    ///
264    /// This will also push any preceding or inline trivia into the stream.
265    /// Any token may have preceding or inline trivia, unless that token is
266    /// itself trivia (i.e. trivia cannot have trivia).
267    ///
268    /// # Panics
269    ///
270    /// This will panic if the provided token is trivia.
271    pub fn push_ast_token(&mut self, token: &wdl_ast::Token) {
272        self.push_preceding_trivia(token);
273        self.0.push(PreToken::Literal(
274            Rc::new(token.inner().text().to_owned()),
275            token.inner().kind(),
276        ));
277        self.push_inline_trivia(token);
278    }
279
280    /// Pushes a literal string into the stream in place of an AST token.
281    ///
282    /// This will insert any trivia that would have been inserted with the AST
283    /// token.
284    ///
285    /// # Panics
286    ///
287    /// This will panic if the provided token is trivia.
288    pub fn push_literal_in_place_of_token(&mut self, token: &wdl_ast::Token, replacement: String) {
289        self.push_preceding_trivia(token);
290        self.0.push(PreToken::Literal(
291            Rc::new(replacement),
292            token.inner().kind(),
293        ));
294        self.push_inline_trivia(token);
295    }
296
297    /// Pushes a literal string into the stream.
298    ///
299    /// This will not insert any trivia.
300    pub fn push_literal(&mut self, value: String, kind: SyntaxKind) {
301        self.0.push(PreToken::Literal(Rc::new(value), kind));
302    }
303}
wdl_format/token/pre.rs

wdl_format/token/
pre.rs
