wdl_format/token/
pre.rs

1//! Tokens emitted during the formatting of particular elements.
2
3use std::collections::HashSet;
4use std::rc::Rc;
5
6use wdl_ast::AstToken;
7use wdl_ast::DOC_COMMENT_PREFIX;
8use wdl_ast::Directive;
9use wdl_ast::SyntaxKind;
10use wdl_ast::SyntaxTokenExt;
11
12use crate::Comment;
13use crate::Token;
14use crate::TokenStream;
15use crate::Trivia;
16use crate::TriviaBlankLineSpacingPolicy;
17
18/// A token that can be written by elements.
19///
20/// These are tokens that are intended to be written directly by elements to a
21/// [`TokenStream`](super::TokenStream) consisting of [`PreToken`]s. Note that
22/// this will transformed into a [`TokenStream`](super::TokenStream) of
23/// [`PostToken`](super::PostToken)s by a
24/// [`Postprocessor`](super::Postprocessor) (authors of elements are never
25/// expected to write [`PostToken`](super::PostToken)s directly).
26#[derive(Clone, Debug, Eq, PartialEq)]
27pub enum PreToken {
28    /// A non-trivial blank line.
29    ///
30    /// This will not be ignored by the postprocessor (unlike
31    /// [`Trivia::BlankLine`] which is potentially ignored).
32    BlankLine,
33
34    /// The end of a line.
35    LineEnd,
36
37    /// The end of a word.
38    WordEnd,
39
40    /// The start of an indented block.
41    IndentStart,
42
43    /// The end of an indented block.
44    IndentEnd,
45
46    /// How to handle trivial blank lines from this point onwards.
47    LineSpacingPolicy(TriviaBlankLineSpacingPolicy),
48
49    /// Literal text.
50    Literal(Rc<String>, SyntaxKind),
51
52    /// Trivia.
53    Trivia(Trivia),
54
55    /// A temporary indent start. Used in command section formatting.
56    ///
57    /// Command sections must account for indentation from both the
58    /// WDL context and the embedded Bash context, so this is used to
59    /// add additional indentation from the Bash context.
60    TempIndentStart(Rc<String>),
61
62    /// A temporary indent end. Used in command section formatting.
63    ///
64    /// See [`PreToken::TempIndentStart`] for more information.
65    TempIndentEnd,
66}
67
68impl std::fmt::Display for PreToken {
69    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
70        match self {
71            PreToken::BlankLine => write!(f, "<BlankLine>"),
72            PreToken::LineEnd => write!(f, "<EndOfLine>"),
73            PreToken::WordEnd => write!(f, "<WordEnd>"),
74            PreToken::IndentStart => write!(f, "<IndentStart>"),
75            PreToken::IndentEnd => write!(f, "<IndentEnd>"),
76            PreToken::LineSpacingPolicy(policy) => {
77                write!(f, "<LineSpacingPolicy@{policy:?}>")
78            }
79            PreToken::Literal(value, kind) => {
80                write!(f, "<Literal-{kind:?}@{value}>")
81            }
82            PreToken::Trivia(trivia) => match trivia {
83                Trivia::BlankLine => {
84                    write!(f, "<OptionalBlankLine>")
85                }
86                Trivia::Comment(comment) => match comment {
87                    Comment::Directive(directive) => {
88                        write!(f, "<Comment-Directive@{directive:?}>")
89                    }
90                    Comment::Documentation(documentation) => {
91                        write!(f, "<Comment-Documentation@{documentation}>")
92                    }
93                    Comment::Preceding(value) => {
94                        write!(f, "<Comment-Preceding@{value}>")
95                    }
96                    Comment::Inline(value) => {
97                        write!(f, "<Comment-Inline@{value}>")
98                    }
99                },
100            },
101            PreToken::TempIndentStart(value) => write!(f, "<TempIndentStart@{value}>"),
102            PreToken::TempIndentEnd => write!(f, "<TempIndentEnd>"),
103        }
104    }
105}
106
107impl Token for PreToken {
108    /// Returns a displayable version of the token.
109    fn display<'a>(&'a self, _config: &'a crate::Config) -> impl std::fmt::Display {
110        self
111    }
112}
113
114impl TokenStream<PreToken> {
115    /// Inserts a blank line token to the stream if the stream does not already
116    /// end with a blank line. This will replace any [`Trivia::BlankLine`]
117    /// tokens with [`PreToken::BlankLine`].
118    pub fn blank_line(&mut self) {
119        self.trim_while(|t| matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine)));
120        self.0.push(PreToken::BlankLine);
121    }
122
123    /// Inserts an end of line token to the stream if the stream does not
124    /// already end with an end of line token.
125    ///
126    /// This will also trim any trailing [`PreToken::WordEnd`] tokens.
127    pub fn end_line(&mut self) {
128        self.trim_while(|t| matches!(t, PreToken::WordEnd | PreToken::LineEnd));
129        self.0.push(PreToken::LineEnd);
130    }
131
132    /// Inserts a word end token to the stream if the stream does not already
133    /// end with a word end token.
134    pub fn end_word(&mut self) {
135        self.trim_end(&PreToken::WordEnd);
136        self.0.push(PreToken::WordEnd);
137    }
138
139    /// Inserts an indent start token to the stream. This will also end the
140    /// current line.
141    pub fn increment_indent(&mut self) {
142        self.end_line();
143        self.0.push(PreToken::IndentStart);
144    }
145
146    /// Inserts an indent end token to the stream. This will also end the
147    /// current line.
148    pub fn decrement_indent(&mut self) {
149        self.end_line();
150        self.0.push(PreToken::IndentEnd);
151    }
152
153    /// Inserts a trivial blank lines "always allowed" context change.
154    pub fn allow_blank_lines(&mut self) {
155        self.0.push(PreToken::LineSpacingPolicy(
156            TriviaBlankLineSpacingPolicy::Always,
157        ));
158    }
159
160    /// Inserts a trivial blank lines "not allowed after comments" context
161    /// change.
162    pub fn ignore_trailing_blank_lines(&mut self) {
163        self.0.push(PreToken::LineSpacingPolicy(
164            TriviaBlankLineSpacingPolicy::RemoveTrailingBlanks,
165        ));
166    }
167
168    /// Inserts any preceding trivia into the stream.
169    ///
170    /// This will consolidate all doc comments and directive comments which
171    /// precede this token.
172    ///
173    /// # Panics
174    ///
175    /// This will panic if the provided token is itself trivia, as trivia
176    /// cannot have trivia.
177    fn push_preceding_trivia(&mut self, token: &wdl_ast::Token) {
178        assert!(!token.inner().kind().is_trivia());
179        let preceding_trivia = token.inner().preceding_trivia();
180        let mut documentation = String::new();
181        let mut trivia = Vec::new();
182        let mut exceptions = HashSet::new();
183        for token in preceding_trivia {
184            match token.kind() {
185                SyntaxKind::Whitespace => {
186                    if !self.0.last().is_some_and(|t| {
187                        matches!(t, PreToken::BlankLine | PreToken::Trivia(Trivia::BlankLine))
188                    }) {
189                        trivia.push(PreToken::Trivia(Trivia::BlankLine));
190                    }
191                }
192                SyntaxKind::Comment => {
193                    if let Some(t) = token.text().strip_prefix(DOC_COMMENT_PREFIX) {
194                        // do not `trim()` the token as the whitespace may
195                        // have syntactical meaning in markdown
196                        documentation.push_str(t);
197                        documentation.push('\n');
198                    } else if let Some(comment) = wdl_ast::Comment::cast(token.clone())
199                        && let Some(directive) = comment.directive()
200                    {
201                        match directive {
202                            Directive::Except(e) => exceptions.extend(e),
203                        }
204                    } else {
205                        let comment = PreToken::Trivia(Trivia::Comment(Comment::Preceding(
206                            Rc::new(token.text().trim_end().to_string()),
207                        )));
208                        trivia.push(comment);
209                    }
210                }
211                _ => unreachable!("unexpected trivia: {:?}", token),
212            };
213        }
214
215        let mut trivia = trivia.into_iter().peekable();
216        // Preserve any leading blank lines
217        if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
218            self.0.push(trivia.next().unwrap());
219        }
220        let mut docs_present = false;
221        if !documentation.is_empty() {
222            docs_present = true;
223            let comment = PreToken::Trivia(Trivia::Comment(Comment::Documentation(Rc::new(
224                documentation,
225            ))));
226            self.0.push(comment);
227
228            // don't allow documentation to "float" above the item being documented
229            if let Some(PreToken::Trivia(Trivia::BlankLine)) = trivia.peek() {
230                let _ = trivia.next();
231            }
232        }
233        for token in trivia {
234            self.0.push(token);
235        }
236        if docs_present && let Some(PreToken::Trivia(Trivia::BlankLine)) = self.0.last() {
237            // don't allow documentation to "float" above the item being documented
238            self.0.pop();
239        }
240        if !exceptions.is_empty() {
241            let comment = PreToken::Trivia(Trivia::Comment(Comment::Directive(Rc::new(
242                Directive::Except(exceptions),
243            ))));
244            self.0.push(comment);
245        }
246    }
247
248    /// Inserts any inline trivia into the stream.
249    ///
250    /// # Panics
251    ///
252    /// This will panic if the provided token is itself trivia, as trivia
253    /// cannot have trivia.
254    fn push_inline_trivia(&mut self, token: &wdl_ast::Token) {
255        assert!(!token.inner().kind().is_trivia());
256        if let Some(token) = token.inner().inline_comment() {
257            let inline_comment = PreToken::Trivia(Trivia::Comment(Comment::Inline(Rc::new(
258                token.text().trim_end().to_owned(),
259            ))));
260            self.0.push(inline_comment);
261        }
262    }
263
264    /// Pushes an AST token into the stream.
265    ///
266    /// This will also push any preceding or inline trivia into the stream.
267    /// Any token may have preceding or inline trivia, unless that token is
268    /// itself trivia (i.e. trivia cannot have trivia).
269    ///
270    /// # Panics
271    ///
272    /// This will panic if the provided token is trivia.
273    pub fn push_ast_token(&mut self, token: &wdl_ast::Token) {
274        self.push_preceding_trivia(token);
275        self.0.push(PreToken::Literal(
276            Rc::new(token.inner().text().to_owned()),
277            token.inner().kind(),
278        ));
279        self.push_inline_trivia(token);
280    }
281
282    /// Pushes a literal string into the stream in place of an AST token.
283    ///
284    /// This will insert any trivia that would have been inserted with the AST
285    /// token.
286    ///
287    /// # Panics
288    ///
289    /// This will panic if the provided token is trivia.
290    pub fn push_literal_in_place_of_token(&mut self, token: &wdl_ast::Token, replacement: String) {
291        self.push_preceding_trivia(token);
292        self.0.push(PreToken::Literal(
293            Rc::new(replacement),
294            token.inner().kind(),
295        ));
296        self.push_inline_trivia(token);
297    }
298
299    /// Pushes a literal string into the stream.
300    ///
301    /// This will not insert any trivia.
302    pub fn push_literal(&mut self, value: String, kind: SyntaxKind) {
303        self.0.push(PreToken::Literal(Rc::new(value), kind));
304    }
305}
wdl_format/token/pre.rs

wdl_format/token/
pre.rs
