wdl_format/token/
post.rs

1//! Postprocessed tokens.
2//!
3//! Generally speaking, unless you are working with the internals of code
4//! formatting, you're not going to be working with these.
5
6use std::fmt::Display;
7use std::rc::Rc;
8
9use wdl_ast::DIRECTIVE_COMMENT_PREFIX;
10use wdl_ast::DIRECTIVE_DELIMITER;
11use wdl_ast::DOC_COMMENT_PREFIX;
12use wdl_ast::Directive;
13use wdl_ast::SyntaxKind;
14
15use crate::Comment;
16use crate::Config;
17use crate::Indent;
18use crate::PreToken;
19use crate::SPACE;
20use crate::Token;
21use crate::TokenStream;
22use crate::Trivia;
23use crate::TriviaBlankLineSpacingPolicy;
24
25/// [`PostToken`]s that precede an inline comment.
26const INLINE_COMMENT_PRECEDING_TOKENS: [PostToken; 2] = [PostToken::Space, PostToken::Space];
27
28/// A postprocessed token.
29#[derive(Clone, Eq, PartialEq)]
30pub enum PostToken {
31    /// A space.
32    Space,
33
34    /// A newline.
35    Newline,
36
37    /// One indentation.
38    Indent,
39
40    /// A temporary indent.
41    ///
42    /// This is added after a [`PostToken::Indent`] during the formatting of
43    /// command sections.
44    TempIndent(Rc<String>),
45
46    /// A string literal.
47    Literal(Rc<String>),
48
49    /// A doc comment block.
50    Documentation {
51        /// The current indent level.
52        num_indents: usize,
53        /// The contents of the doc comment block.
54        contents: Rc<String>,
55    },
56
57    /// A directive comment.
58    Directive {
59        /// The current indent level.
60        num_indents: usize,
61        /// The directive.
62        directive: Rc<Directive>,
63    },
64}
65
66impl std::fmt::Debug for PostToken {
67    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68        match self {
69            Self::Space => write!(f, "<SPACE>"),
70            Self::Newline => write!(f, "<NEWLINE>"),
71            Self::Indent => write!(f, "<INDENT>"),
72            Self::TempIndent(value) => write!(f, "<TEMP_INDENT@{value}>"),
73            Self::Literal(value) => write!(f, "<LITERAL@{value}>"),
74            Self::Directive { directive, .. } => write!(f, "<DIRECTIVE@{directive:?}>"),
75            Self::Documentation { contents, .. } => write!(f, "<DOCUMENTATION@{contents}>"),
76        }
77    }
78}
79
80impl Token for PostToken {
81    /// Returns a displayable version of the token.
82    fn display<'a>(&'a self, config: &'a Config) -> impl Display + 'a {
83        /// A displayable version of a [`PostToken`].
84        struct Display<'a> {
85            /// The token to display.
86            token: &'a PostToken,
87            /// The configuration to use.
88            config: &'a Config,
89        }
90
91        fn write_indents(
92            f: &mut std::fmt::Formatter<'_>,
93            indent: &Indent,
94            num_indents: usize,
95        ) -> std::fmt::Result {
96            for _ in 0usize..num_indents {
97                write!(f, "{indent}")?;
98            }
99            Ok(())
100        }
101
102        impl std::fmt::Display for Display<'_> {
103            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104                match self.token {
105                    PostToken::Space => write!(f, "{SPACE}"),
106                    PostToken::Newline => write!(f, "{}", self.config.newline_style.as_str()),
107                    PostToken::Indent => {
108                        write!(f, "{indent}", indent = self.config.indent)
109                    }
110                    PostToken::TempIndent(value) => write!(f, "{value}"),
111                    PostToken::Literal(value) => write!(f, "{value}"),
112                    PostToken::Documentation {
113                        num_indents,
114                        contents: markdown,
115                    } => {
116                        let prefix = DOC_COMMENT_PREFIX;
117                        write!(f, "{prefix}")?;
118                        let mut lines = markdown.lines().peekable();
119                        while let Some(cur) = lines.next() {
120                            write!(f, "{cur}")?;
121                            if lines.peek().is_some() {
122                                write!(f, "{}", self.config.newline_style.as_str())?;
123                                write_indents(f, &self.config.indent, *num_indents)?;
124                                write!(f, "{prefix}")?;
125                            }
126                        }
127                        Ok(())
128                    }
129                    PostToken::Directive {
130                        num_indents,
131                        directive,
132                    } => {
133                        let mut prefix = format!("{} ", DIRECTIVE_COMMENT_PREFIX);
134                        match &**directive {
135                            Directive::Except(exceptions) => {
136                                prefix.push_str("except");
137                                prefix.push_str(DIRECTIVE_DELIMITER);
138                                prefix.push(' ');
139                                let mut rules: Vec<String> =
140                                    exceptions.iter().map(|e| e.name.clone()).collect();
141                                rules.sort();
142                                write!(f, "{prefix}")?;
143                                if let Some(max) = self.config.max_line_length.get() {
144                                    let indent_width = self.config.indent.num() * num_indents;
145                                    let start_width = indent_width + prefix.len();
146                                    let mut remaining = max.saturating_sub(start_width);
147                                    let mut written_to_cur_line = 0usize;
148                                    for rule in rules {
149                                        let cur_len = rule.len();
150                                        if written_to_cur_line == 0 {
151                                            write!(f, "{rule}")?;
152                                            remaining = remaining.saturating_sub(cur_len);
153                                            written_to_cur_line += 1;
154                                        } else if remaining.saturating_sub(cur_len + 2) > 0 {
155                                            // NOTE: the `+ 2` accounts for
156                                            // the `", "` separator written
157                                            // before each subsequent rule.
158                                            write!(f, ", {rule}")?;
159                                            remaining = remaining.saturating_sub(cur_len + 2);
160                                            written_to_cur_line += 1;
161                                        } else {
162                                            // Current rule does not fit
163                                            write!(f, "{}", self.config.newline_style.as_str())?;
164                                            write_indents(f, &self.config.indent, *num_indents)?;
165                                            write!(f, "{prefix}{rule}")?;
166                                            written_to_cur_line = 1;
167                                            remaining = max.saturating_sub(start_width + cur_len);
168                                        }
169                                    }
170                                    Ok(())
171                                } else {
172                                    write!(f, "{rules}", rules = rules.join(", "))
173                                }
174                            }
175                        }
176                    }
177                }
178            }
179        }
180
181        Display {
182            token: self,
183            config,
184        }
185    }
186}
187
188impl PostToken {
189    /// Gets the width of the [`PostToken`].
190    ///
191    /// This is used to determine how much space the token takes up _within a
192    /// single line_ for the purposes of respecting the maximum line length.
193    /// As such, newlines are considered zero-width tokens. Similarly, doc
194    /// comments and directive comments are considered zero-width as they always
195    /// appear on their own lines.
196    fn width(&self, config: &crate::Config) -> usize {
197        match self {
198            Self::Space => SPACE.len(), // 1 character
199            Self::Newline => 0,
200            Self::Indent => config.indent.num(),
201            Self::TempIndent(value) => value.len(),
202            Self::Literal(value) => value.len(),
203            Self::Directive { .. } => 0,
204            Self::Documentation { .. } => 0,
205        }
206    }
207}
208
209impl TokenStream<PostToken> {
210    /// Gets the maximum width of the [`TokenStream`].
211    ///
212    /// This is suitable to call if the stream represents multiple lines.
213    #[expect(dead_code)]
214    fn max_width(&self, config: &Config) -> usize {
215        let mut max: usize = 0;
216        let mut cur_width: usize = 0;
217        for token in self.iter() {
218            cur_width += token.width(config);
219            if token == &PostToken::Newline {
220                max = max.max(cur_width);
221                cur_width = 0;
222            }
223        }
224        max.max(cur_width)
225    }
226
227    /// Gets the width of the last line of the [`TokenStream`].
228    fn last_line_width(&self, config: &Config) -> usize {
229        let mut width = 0;
230        for token in self.iter().rev() {
231            if token == &PostToken::Newline {
232                break;
233            }
234            width += token.width(config);
235        }
236        width
237    }
238}
239
240/// A line break.
241enum LineBreak {
242    /// A line break that can be inserted before a token.
243    Before,
244    /// A line break that can be inserted after a token.
245    After,
246}
247
248/// Returns whether a token can be line broken.
249///
250/// Note that this function only returns `true` for tokens that do not otherwise
251/// get linebroken. Tokens which are either always or sometimes linebroken
252/// should be handled during [`PreToken`] processing.
253fn can_be_line_broken(kind: SyntaxKind) -> Option<LineBreak> {
254    match kind {
255        SyntaxKind::CloseBracket
256        | SyntaxKind::CloseParen
257        | SyntaxKind::Assignment
258        | SyntaxKind::Plus
259        | SyntaxKind::Minus
260        | SyntaxKind::Asterisk
261        | SyntaxKind::Slash
262        | SyntaxKind::Percent
263        | SyntaxKind::Exponentiation
264        | SyntaxKind::Equal
265        | SyntaxKind::NotEqual
266        | SyntaxKind::Less
267        | SyntaxKind::LessEqual
268        | SyntaxKind::Greater
269        | SyntaxKind::GreaterEqual
270        | SyntaxKind::LogicalAnd
271        | SyntaxKind::LogicalOr
272        | SyntaxKind::AfterKeyword
273        | SyntaxKind::AsKeyword => Some(LineBreak::Before),
274        SyntaxKind::OpenBracket
275        | SyntaxKind::OpenParen
276        | SyntaxKind::Colon
277        | SyntaxKind::PlaceholderOpen
278        | SyntaxKind::Comma => Some(LineBreak::After),
279        _ => None,
280    }
281}
282
283/// Gets the corresponding [`SyntaxKind`] that should be line broken in tandem
284/// with the provided [`SyntaxKind`].
285fn tandem_line_break(kind: SyntaxKind) -> Option<SyntaxKind> {
286    match kind {
287        SyntaxKind::OpenBrace => Some(SyntaxKind::CloseBrace),
288        SyntaxKind::OpenBracket => Some(SyntaxKind::CloseBracket),
289        SyntaxKind::OpenParen => Some(SyntaxKind::CloseParen),
290        SyntaxKind::OpenHeredoc => Some(SyntaxKind::CloseHeredoc),
291        SyntaxKind::PlaceholderOpen => Some(SyntaxKind::CloseBrace),
292        _ => None,
293    }
294}
295
296/// Tokens that should be allowed to be interrupted.
297///
298/// If one of these tokens comes after an interruption, the typical extra level
299/// of indentation that appears after interrupts won't be inserted.
300fn allow_interruption(kind: SyntaxKind) -> bool {
301    matches!(
302        kind,
303        SyntaxKind::OpenBrace
304            | SyntaxKind::OpenBracket
305            | SyntaxKind::OpenParen
306            | SyntaxKind::OpenHeredoc
307            | SyntaxKind::CloseBrace
308            | SyntaxKind::CloseBracket
309            | SyntaxKind::CloseParen
310            | SyntaxKind::CloseHeredoc
311            | SyntaxKind::IfKeyword
312            | SyntaxKind::ElseKeyword
313    )
314}
315
316/// Tracks a tandem break.
317struct TandemBreak {
318    /// The [`SyntaxKind`] which opened this tandem break.
319    pub open: SyntaxKind,
320    /// The [`SyntaxKind`] which will close this tandem break.
321    pub close: SyntaxKind,
322    /// Token depth since opening the break.
323    ///
324    /// The close break is only added when `depth == 0`.
325    /// This is incremented by one for every token matching `open` after the
326    /// break is initiated. It is decremented by one for every token
327    /// matching `close` after the break is initiated.
328    pub depth: usize,
329}
330
331/// Current position in a line.
332#[derive(Copy, Clone, Default, Eq, PartialEq)]
333enum LinePosition {
334    /// The start of a line.
335    #[default]
336    StartOfLine,
337
338    /// The middle of a line.
339    MiddleOfLine,
340}
341
342/// A postprocessor of [tokens](PreToken).
343#[derive(Clone, Default)]
344pub struct Postprocessor {
345    /// The current position in the line.
346    position: LinePosition,
347
348    /// The current indentation level.
349    indent_level: usize,
350
351    /// Whether the current line has been interrupted by trivia.
352    interrupted: bool,
353
354    /// The current trivial blank line spacing policy.
355    line_spacing_policy: TriviaBlankLineSpacingPolicy,
356
357    /// Temporary indentation to add.
358    temp_indent: Option<Rc<String>>,
359
360    /// Whether potential splits should be linebroken or not.
361    fit_potential_splits: bool,
362
363    /// The delimiter to use when "fitting" a potential split.
364    fit_delimiter: Option<Rc<String>>,
365}
366
367impl Postprocessor {
368    /// Runs the postprocessor.
369    pub fn run(&mut self, input: TokenStream<PreToken>, config: &Config) -> TokenStream<PostToken> {
370        let mut output = TokenStream::<PostToken>::default();
371        let mut buffer = TokenStream::<PreToken>::default();
372
373        for token in input {
374            if matches!(token, PreToken::LineEnd) {
375                buffer.push(token);
376                self.flush(&buffer, &mut output, config);
377                self.trim_whitespace(&mut output);
378                output.push(PostToken::Newline);
379
380                buffer.clear();
381                self.interrupted = false;
382            } else {
383                buffer.push(token);
384            }
385        }
386
387        output
388    }
389
390    /// Takes a step of a [`PreToken`] stream and processes the appropriate
391    /// [`PostToken`]s.
392    fn step(
393        &mut self,
394        token: PreToken,
395        next: Option<&PreToken>,
396        stream: &mut TokenStream<PostToken>,
397    ) {
398        if stream.is_empty() {
399            self.indent(stream);
400        }
401        match token {
402            PreToken::BlankLine => {
403                self.blank_line(stream);
404            }
405            PreToken::LineEnd => {
406                self.interrupted = false;
407                self.end_line(stream);
408            }
409            PreToken::WordEnd => {
410                stream.trim_end(&PostToken::Space);
411
412                if self.position == LinePosition::MiddleOfLine {
413                    stream.push(PostToken::Space);
414                } else {
415                    // We're at the start of a line, so we don't need to add a
416                    // space.
417                }
418            }
419            PreToken::IndentStart => {
420                self.indent_level += 1;
421            }
422            PreToken::IndentEnd => {
423                self.indent_level = self.indent_level.saturating_sub(1);
424            }
425            PreToken::LineSpacingPolicy(policy) => {
426                self.line_spacing_policy = policy;
427            }
428            PreToken::Literal(value, kind) => {
429                assert!(!kind.is_trivia());
430
431                // This is special handling for inserting the empty string.
432                // We remove any indentation or spaces from the end of the
433                // stream before adding the empty string as a literal.
434                if value.is_empty() {
435                    self.trim_last_line(stream);
436                }
437
438                if self.interrupted && allow_interruption(kind) {
439                    self.pop_indent(stream);
440                }
441
442                stream.push(PostToken::Literal(value));
443                self.position = LinePosition::MiddleOfLine;
444            }
445            PreToken::Trivia(trivia) => match trivia {
446                Trivia::BlankLine => match self.line_spacing_policy {
447                    TriviaBlankLineSpacingPolicy::Always => {
448                        self.blank_line(stream);
449                    }
450                    TriviaBlankLineSpacingPolicy::RemoveTrailingBlanks => {
451                        if matches!(next, Some(&PreToken::Trivia(Trivia::Comment(_)))) {
452                            self.blank_line(stream);
453                        }
454                    }
455                },
456                Trivia::Comment(comment) => {
457                    match comment {
458                        Comment::Preceding(value) => {
459                            if self.position == LinePosition::MiddleOfLine {
460                                self.interrupted = true;
461                                self.end_line(stream);
462                                if let Some(PreToken::Literal(_, next_kind)) = next
463                                    && allow_interruption(*next_kind)
464                                {
465                                    self.pop_indent(stream);
466                                }
467                            }
468                            stream.push(PostToken::Literal(value));
469                        }
470                        Comment::Inline(value) => {
471                            assert!(self.position == LinePosition::MiddleOfLine);
472                            if let Some(next) = next
473                                && !matches!(next, &PreToken::LineEnd)
474                            {
475                                self.interrupted = true;
476                            }
477                            self.trim_last_line(stream);
478                            for token in INLINE_COMMENT_PRECEDING_TOKENS.iter() {
479                                stream.push(token.clone());
480                            }
481                            stream.push(PostToken::Literal(value));
482                        }
483                        Comment::Documentation(contents) => {
484                            if self.position == LinePosition::MiddleOfLine {
485                                self.interrupted = true;
486                                self.end_line(stream);
487                            }
488                            stream.push(PostToken::Documentation {
489                                num_indents: self.indent_level,
490                                contents,
491                            });
492                        }
493                        Comment::Directive(directive) => {
494                            if self.position == LinePosition::MiddleOfLine {
495                                self.interrupted = true;
496                                self.end_line(stream);
497                            }
498                            stream.push(PostToken::Directive {
499                                num_indents: self.indent_level,
500                                directive,
501                            });
502                        }
503                    }
504                    self.position = LinePosition::MiddleOfLine;
505                    self.end_line(stream);
506                }
507            },
508            PreToken::TempIndentStart(bash_indent) => {
509                self.temp_indent = Some(bash_indent);
510            }
511            PreToken::TempIndentEnd => {
512                self.temp_indent = None;
513            }
514            PreToken::FitOrSplitStart {
515                fit_start,
516                fit_delimiter,
517                split_end_line,
518            } => {
519                if self.fit_potential_splits {
520                    stream.push(PostToken::Literal(fit_start));
521                    self.fit_delimiter = Some(fit_delimiter);
522                } else if split_end_line {
523                    self.end_line(stream);
524                }
525            }
526            PreToken::PotentialSplit => {
527                if self.fit_potential_splits
528                    && let Some(delim) = &self.fit_delimiter
529                {
530                    self.trim_last_line(stream);
531                    stream.push(PostToken::Literal(delim.clone()));
532                } else {
533                    self.end_line(stream);
534                }
535            }
536            PreToken::FitOrSplitEnd {
537                fit_end,
538                split_end,
539                split_end_line,
540            } => {
541                if self.fit_potential_splits {
542                    if fit_end.is_empty() {
543                        self.trim_last_line(stream);
544                    }
545                    stream.push(PostToken::Literal(fit_end));
546                } else {
547                    if split_end.is_empty() {
548                        self.trim_last_line(stream);
549                    }
550                    stream.push(PostToken::Literal(split_end));
551                    if split_end_line {
552                        self.end_line(stream);
553                    }
554                }
555                self.fit_delimiter = None;
556            }
557        }
558    }
559
560    /// Flushes the `in_stream` buffer to the `out_stream`.
561    fn flush(
562        &mut self,
563        in_stream: &TokenStream<PreToken>,
564        out_stream: &mut TokenStream<PostToken>,
565        config: &Config,
566    ) {
567        assert!(!self.interrupted);
568        assert!(
569            self.position == LinePosition::StartOfLine,
570            "`flush()` must be called from the start of a line"
571        );
572        let init_self = self.clone();
573
574        let max_length = config.max_line_length.get();
575
576        let mut break_stack: Vec<TandemBreak> = Vec::new();
577        let mut prev_kind = None;
578
579        let mut pre_buffer = in_stream.iter().enumerate().peekable();
580        let mut post_buffer = TokenStream::<PostToken>::default();
581
582        // If we encounter any potential splits, we use the first iteration to
583        // find any spans that will fit. If we don't find any potential
584        // splits, we can just push the result of the first iteration to
585        // the out stream and be done.
586        self.fit_potential_splits = true;
587        let mut fit_spans = Vec::new();
588        let mut fit_start = None;
589        let mut reprocess_needed = false;
590
591        // First iteration
592        while let Some((i, token)) = pre_buffer.next() {
593            match token {
594                PreToken::FitOrSplitStart { .. } => {
595                    reprocess_needed = true;
596                    // overwrite any prior start. Only the innermost span is a
597                    // candidate for fitting.
598                    fit_start = Some(i);
599                }
600                PreToken::PotentialSplit => {
601                    reprocess_needed = true;
602                }
603                PreToken::FitOrSplitEnd { .. } => {
604                    if let Some(start) = fit_start.take()
605                        && max_length.is_none_or(|max| post_buffer.last_line_width(config) < max)
606                        && !self.interrupted
607                    {
608                        fit_spans.push((start, i));
609                    }
610                    reprocess_needed = true;
611                }
612                _ => {}
613            }
614
615            let mut cache = None;
616            let mut cached_self = None;
617            let mut cached_on = None;
618
619            if let Some(max) = max_length
620                && !reprocess_needed
621            {
622                if let PreToken::Literal(_, kind) = token {
623                    // Check if we need a break to match a prior tandem break
624                    if let Some(top_of_stack) = break_stack.last_mut() {
625                        if *kind == top_of_stack.close {
626                            if top_of_stack.depth > 0 {
627                                top_of_stack.depth -= 1;
628                            } else {
629                                break_stack.pop();
630                                self.indent_level -= 1;
631                                self.end_line(&mut post_buffer);
632                            }
633                        } else if *kind == top_of_stack.open {
634                            top_of_stack.depth += 1;
635                        }
636                    }
637                    if let Some(LineBreak::Before) = can_be_line_broken(*kind) {
638                        if post_buffer.last_line_width(config) > max {
639                            // the line is already too long
640                            self.interrupted = true;
641                            self.end_line(&mut post_buffer);
642                            if let Some(also_break_on) = tandem_line_break(*kind) {
643                                self.indent_level += 1;
644                                self.interrupted = false;
645                                let tandem_break = TandemBreak {
646                                    open: *kind,
647                                    close: also_break_on,
648                                    depth: 0,
649                                };
650                                break_stack.push(tandem_break);
651                            }
652                        } else {
653                            // cache the current state so we can revert to it if
654                            // the line is too long after the next step.
655                            cache = Some(post_buffer.clone());
656                            cached_self = Some(self.clone());
657                            cached_on = Some(*kind);
658                        }
659                    } else if let Some(k) = prev_kind.take()
660                        && matches!(can_be_line_broken(k), Some(LineBreak::After))
661                    {
662                        cache = Some(post_buffer.clone());
663                        cached_self = Some(self.clone());
664                        cached_on = Some(k);
665                    }
666                }
667                prev_kind = None;
668            }
669
670            let next = pre_buffer.peek().map(|(_, n)| *n);
671            self.step(token.clone(), next, &mut post_buffer);
672
673            // If we cached before the step and the line is now too long,
674            // revert, line break, then repeat the step we just
675            // took.
676            if max_length.is_some_and(|max| post_buffer.last_line_width(config) > max)
677                && let Some(cache) = cache.take()
678                && let Some(cached_self) = cached_self.take()
679                && let Some(cached_on) = cached_on.take()
680                && !reprocess_needed
681            {
682                // revert
683                post_buffer = cache;
684                // reset self
685                *self = cached_self;
686
687                // line break
688                self.interrupted = true;
689                self.end_line(&mut post_buffer);
690                if let Some(also_break_on) = tandem_line_break(cached_on) {
691                    self.indent_level += 1;
692                    self.interrupted = false;
693                    let tandem_break = TandemBreak {
694                        open: cached_on,
695                        close: also_break_on,
696                        depth: 0,
697                    };
698                    break_stack.push(tandem_break);
699                }
700
701                // repeat step
702                self.step(token.clone(), next, &mut post_buffer);
703            }
704
705            // check if we should line break now, after the step has been taken
706            if let Some(max) = max_length
707                && let PreToken::Literal(_, kind) = token
708                && !reprocess_needed
709            {
710                // will be cleared to `None` if a linebreak is added this pass
711                prev_kind = Some(*kind);
712
713                // Check if we need a break to match a prior tandem break
714                if let Some(top_of_stack) = break_stack.last_mut() {
715                    if *kind == top_of_stack.close {
716                        if top_of_stack.depth > 0 {
717                            top_of_stack.depth -= 1;
718                        } else {
719                            break_stack.pop();
720                            self.indent_level -= 1;
721                            self.end_line(&mut post_buffer);
722
723                            prev_kind = None;
724                        }
725                    } else if *kind == top_of_stack.open {
726                        top_of_stack.depth += 1;
727                    }
728                }
729
730                if let Some(LineBreak::After) = can_be_line_broken(*kind)
731                    && post_buffer.last_line_width(config) > max
732                {
733                    self.interrupted = true;
734                    self.end_line(&mut post_buffer);
735                    if let Some(also_break_on) = tandem_line_break(*kind) {
736                        self.indent_level += 1;
737                        self.interrupted = false;
738                        let tandem_break = TandemBreak {
739                            open: *kind,
740                            close: also_break_on,
741                            depth: 0,
742                        };
743                        break_stack.push(tandem_break);
744                    }
745
746                    prev_kind = None;
747                }
748            }
749        }
750
751        // Any tandem break still open at the end of the flush will never be
752        // closed, so unwind the indentation it introduced.
753        self.indent_level = self.indent_level.saturating_sub(break_stack.len());
754
755        if !reprocess_needed {
756            out_stream.extend(post_buffer);
757            return;
758        }
759
760        *self = init_self;
761
762        break_stack.clear();
763        prev_kind = None;
764
765        pre_buffer = in_stream.iter().enumerate().peekable();
766        post_buffer.clear();
767
768        self.fit_potential_splits = false;
769        let mut fit_spans = fit_spans.into_iter();
770        let mut fit_span = fit_spans.next();
771        let mut indent_on_first_split = false;
772
773        // Second iteration
774        while let Some((i, token)) = pre_buffer.next() {
775            let mut cache = None;
776            let mut cached_self = None;
777            let mut cached_on = None;
778
779            if fit_span.is_some_and(|(start, _end)| start == i) {
780                self.fit_potential_splits = true;
781            } else if let PreToken::FitOrSplitStart { split_end_line, .. } = token {
782                if *split_end_line {
783                    self.indent_level += 1;
784                    self.interrupted = false;
785                } else {
786                    indent_on_first_split = true;
787                }
788            }
789            if !self.fit_potential_splits && matches!(token, PreToken::PotentialSplit) {
790                self.interrupted = false;
791                if indent_on_first_split {
792                    self.indent_level += 1;
793                    indent_on_first_split = false;
794                }
795            }
796            if !self.fit_potential_splits
797                && let PreToken::FitOrSplitEnd { split_end_line, .. } = token
798            {
799                self.indent_level -= 1;
800                if *split_end_line {
801                    self.interrupted = false;
802                }
803            }
804
805            if let Some(max) = max_length
806                && !self.fit_potential_splits
807            {
808                if let PreToken::Literal(_, kind) = token {
809                    // Check if we need a break to match a prior tandem break
810                    if let Some(top_of_stack) = break_stack.last_mut() {
811                        if *kind == top_of_stack.close {
812                            if top_of_stack.depth > 0 {
813                                top_of_stack.depth -= 1;
814                            } else {
815                                break_stack.pop();
816                                self.indent_level -= 1;
817                                self.end_line(&mut post_buffer);
818                            }
819                        } else if *kind == top_of_stack.open {
820                            top_of_stack.depth += 1;
821                        }
822                    }
823                    if let Some(LineBreak::Before) = can_be_line_broken(*kind) {
824                        if post_buffer.last_line_width(config) > max {
825                            // the line is already too long
826                            self.interrupted = true;
827                            self.end_line(&mut post_buffer);
828                            if let Some(also_break_on) = tandem_line_break(*kind) {
829                                self.indent_level += 1;
830                                self.interrupted = false;
831                                let tandem_break = TandemBreak {
832                                    open: *kind,
833                                    close: also_break_on,
834                                    depth: 0,
835                                };
836                                break_stack.push(tandem_break);
837                            }
838                        } else {
839                            // cache the current state so we can revert to it if
840                            // the line is too long after the next step.
841                            cache = Some(post_buffer.clone());
842                            cached_self = Some(self.clone());
843                            cached_on = Some(*kind);
844                        }
845                    } else if let Some(k) = prev_kind.take()
846                        && matches!(can_be_line_broken(k), Some(LineBreak::After))
847                    {
848                        cache = Some(post_buffer.clone());
849                        cached_self = Some(self.clone());
850                        cached_on = Some(k);
851                    }
852                }
853                prev_kind = None;
854            }
855
856            let next = pre_buffer.peek().map(|(_, n)| *n);
857            self.step(token.clone(), next, &mut post_buffer);
858
859            if fit_span.is_some_and(|(_start, end)| end == i) {
860                fit_span = fit_spans.next();
861                self.fit_potential_splits = false;
862            }
863            // If we cached before the step and the line is now too long,
864            // revert, line break, then repeat the step we just
865            // took.
866            if max_length.is_some_and(|max| post_buffer.last_line_width(config) > max)
867                && let Some(cache) = cache.take()
868                && let Some(cached_self) = cached_self.take()
869                && let Some(cached_on) = cached_on.take()
870            {
871                // revert
872                post_buffer = cache;
873                // reset self
874                *self = cached_self;
875
876                // line break
877                self.interrupted = true;
878                self.end_line(&mut post_buffer);
879                if let Some(also_break_on) = tandem_line_break(cached_on) {
880                    self.indent_level += 1;
881                    self.interrupted = false;
882                    let tandem_break = TandemBreak {
883                        open: cached_on,
884                        close: also_break_on,
885                        depth: 0,
886                    };
887                    break_stack.push(tandem_break);
888                }
889
890                // repeat step
891                self.step(token.clone(), next, &mut post_buffer);
892            }
893
894            // check if we should line break now, after the step has been taken
895            if let Some(max) = max_length
896                && let PreToken::Literal(_, kind) = token
897                && !self.fit_potential_splits
898            {
899                // will be cleared to `None` if a linebreak is added this pass
900                prev_kind = Some(*kind);
901
902                // Check if we need a break to match a prior tandem break
903                if let Some(top_of_stack) = break_stack.last_mut() {
904                    if *kind == top_of_stack.close {
905                        if top_of_stack.depth > 0 {
906                            top_of_stack.depth -= 1;
907                        } else {
908                            break_stack.pop();
909                            self.indent_level -= 1;
910                            self.end_line(&mut post_buffer);
911
912                            prev_kind = None;
913                        }
914                    } else if *kind == top_of_stack.open {
915                        top_of_stack.depth += 1;
916                    }
917                }
918
919                if let Some(LineBreak::After) = can_be_line_broken(*kind)
920                    && post_buffer.last_line_width(config) > max
921                {
922                    self.interrupted = true;
923                    self.end_line(&mut post_buffer);
924                    if let Some(also_break_on) = tandem_line_break(*kind) {
925                        self.indent_level += 1;
926                        self.interrupted = false;
927                        let tandem_break = TandemBreak {
928                            open: *kind,
929                            close: also_break_on,
930                            depth: 0,
931                        };
932                        break_stack.push(tandem_break);
933                    }
934
935                    prev_kind = None;
936                }
937            }
938        }
939
940        // Any tandem break still open at the end of the flush will never be
941        // closed, so unwind the indentation it introduced.
942        self.indent_level = self.indent_level.saturating_sub(break_stack.len());
943
944        out_stream.extend(post_buffer);
945    }
946
947    /// Trims any and all whitespace from the end of the stream.
948    fn trim_whitespace(&self, stream: &mut TokenStream<PostToken>) {
949        stream.trim_while(|token| {
950            matches!(
951                token,
952                PostToken::Space
953                    | PostToken::Newline
954                    | PostToken::Indent
955                    | PostToken::TempIndent(_)
956            )
957        });
958    }
959
960    /// Trims spaces and indents (and not newlines) from the end of the stream.
961    fn trim_last_line(&self, stream: &mut TokenStream<PostToken>) {
962        stream.trim_while(|token| {
963            matches!(
964                token,
965                PostToken::Space | PostToken::Indent | PostToken::TempIndent(_)
966            )
967        });
968    }
969
970    /// Ends the current line without resetting the interrupted flag.
971    ///
972    /// Removes any trailing spaces or indents and adds a newline only if state
973    /// is not [`LinePosition::StartOfLine`]. State is then set to
974    /// [`LinePosition::StartOfLine`]. Finally, indentation is added. Safe to
975    /// call multiple times in a row.
976    fn end_line(&mut self, stream: &mut TokenStream<PostToken>) {
977        self.trim_last_line(stream);
978        if self.position != LinePosition::StartOfLine {
979            stream.push(PostToken::Newline);
980        }
981        self.position = LinePosition::StartOfLine;
982        self.indent(stream);
983    }
984
985    /// Pushes the current indentation level to the stream.
986    ///
987    /// This should only be called when the state is
988    /// [`LinePosition::StartOfLine`]. This does not change the state
989    /// and is safe to call multiple times in a row.
990    fn indent(&self, stream: &mut TokenStream<PostToken>) {
991        assert!(self.position == LinePosition::StartOfLine);
992
993        self.trim_last_line(stream);
994
995        let level = if self.interrupted {
996            self.indent_level + 1
997        } else {
998            self.indent_level
999        };
1000
1001        for _ in 0..level {
1002            stream.push(PostToken::Indent);
1003        }
1004
1005        if let Some(ref temp_indent) = self.temp_indent {
1006            stream.push(PostToken::TempIndent(temp_indent.clone()));
1007        }
1008    }
1009
1010    /// Creates a blank line and then indents.
1011    fn blank_line(&mut self, stream: &mut TokenStream<PostToken>) {
1012        self.trim_whitespace(stream);
1013        self.position = LinePosition::StartOfLine;
1014        if !stream.is_empty() {
1015            stream.push(PostToken::Newline);
1016        }
1017        stream.push(PostToken::Newline);
1018        self.indent(stream);
1019    }
1020
1021    /// Pop exactly one indent off the end of the stream.
1022    ///
1023    /// If the last token is not [`PostToken::Indent`] or
1024    /// [`PostToken::TempIndent`], this is a no-op.
1025    fn pop_indent(&mut self, stream: &mut TokenStream<PostToken>) {
1026        if matches!(
1027            stream.0.last(),
1028            Some(&PostToken::Indent) | Some(&PostToken::TempIndent(_))
1029        ) {
1030            let popped = stream.0.pop().unwrap();
1031            // We don't actually want to pop the TempIndent token,
1032            // but rather a regular Indent token before the temp indent.
1033            if matches!(popped, PostToken::TempIndent(_)) {
1034                stream.0.pop_if(|t| matches!(t, PostToken::Indent));
1035                // Restore the popped TempIndent
1036                stream.0.push(popped);
1037            }
1038        }
1039    }
1040}
wdl_format/token/post.rs

wdl_format/token/
post.rs
