libtlafmt 0.4.1

A formatter library for TLA+ specs, core of tlafmt
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
use std::cmp::max;

use crate::{
    helpers::{Indent, INDENT_STR},
    renderer::token_len,
    token::{Position, Token},
};

use super::is_newline;

/// Scan `buf`, searching for comments that appear vertically aligned (same
/// column index) in the input source file and compute the appropriate amount of
/// padding for adjacent comments to either:
///
///   * Maintain the existing vertical position in source
///   * Increase the column index of all adjacent comments to maintain alignment
///
/// The [`Token::Comment`] that are aligned have their [`Position`] updated to
/// specify relative padding used during rendering in order to maintain the
/// above.
pub(super) fn align_comments(buf: &mut Vec<(Token<'_>, Indent)>) {
    let mut candidates = vec![];

    // Look for end-of-line comments in consecutive rows.
    let mut i = 0;
    let mut newline_count = 0;
    let mut last_newline_count = 0;
    while i < buf.len() {
        // If this token is a comment, extract the source position and indent
        // level for it.
        let (pos, _) = match &buf[i] {
            (Token::Comment(_, v), indent) => (*v, indent),
            (Token::Newline, _) => {
                newline_count += 1;
                i += 1;
                continue;
            }
            _ => {
                i += 1;
                continue;
            }
        };

        // Check this comment against the last candidate (if any).
        //
        // If this comment:
        //
        //   * Appears at the end of a line of text
        //   * The previous line also has an end-of-line comment
        //   * These two comments have the same column index
        //   * They appear on consecutive lines in the rendered output
        //
        // Then they become realignment candidates, and their vertical alignment
        // will be preserved after formatting.

        // If the line delta between this comment and the last candidate is >1,
        // OR this comment is not vertically aligned with the previous, OR are
        // not on consecutive lines in the rendered output then process the
        // aligned candidate batch before continuing.
        if candidates
            .last()
            .map(|(_idx, pos)| pos)
            .is_some_and(|v: &crate::token::Position| {
                (v.unwrap_row() + 1) != pos.unwrap_row()
                    || v.unwrap_col() != pos.unwrap_col()
                    || (newline_count - last_newline_count) > 1
            })
        {
            process_candidates(buf, &mut candidates);
            candidates.truncate(0);
        }

        // Remember the line number at which the last candidate was observed.
        last_newline_count = newline_count;

        // Only end-of-line comments (those followed by newlines) are valid for
        // adjacency alignment.
        if buf.get(i + 1).is_some_and(|(v, _)| is_newline(v)) {
            candidates.push((i, pos));
        }

        i += 1;
    }

    process_candidates(buf, &mut candidates);
}

/// Process a set of comments that are vertically aligned in the source and
/// appear in `buf` to set the appropriate amount of padding on their comments
/// in order to maintain vertical alignment after their lines are formatted.
fn process_candidates(buf: &mut [(Token<'_>, Indent)], candidates: &mut [(usize, Position)]) {
    if candidates.len() < 2 {
        return;
    }

    debug_assert!(buf.len() >= candidates.len());

    // Walk backwards from the first candidate index to find the start of the
    // line for the first candidate.
    let start = candidates[0].0;
    debug_assert!(matches!(buf[candidates[0].0].0, Token::Comment(..)));

    let start = (0..start).rev().find(|&i| is_newline(&buf[i].0));
    let start = match start {
        Some(v) => v,
        None => {
            // This can occur if the start of the document is an ERROR node,
            // followed by a comment on two lines.
            //
            // In this case, avoid trying to align an unparsed spec.
            return;
        }
    };

    // Define the exclusive upper bound token index - the last comment (which is
    // guaranteed to terminate the line it is on).
    let end = candidates
        .last()
        .expect("must have candidate for comment alignment")
        .0;
    debug_assert!(matches!(buf[end].0, Token::Comment(..)));
    debug_assert!(is_newline(&buf[end + 1].0));

    // Compute the post-formatting line lengths of each candidate line.
    let lines = Vec::with_capacity(candidates.len());
    let mut max_line = 0; // Maximum observed line length.

    let iter = buf[start..=end].iter();
    let lines = line_len(iter).fold(lines, |mut acc, v| {
        max_line = max(max_line, v);
        acc.push(v);
        acc
    });

    // Invariant: there is exactly one line length computed per candidate.
    debug_assert_eq!(lines.len(), candidates.len());

    // The new column index is the maximum of:
    //
    //   * all line lengths + 1 space.
    //   * the existing column index as appears in the input source code.
    //
    // This causes the location of the comment to match the pre-formatted output
    // UNLESS a now-formatted line pushes past the previous position, in which
    // case all aligned comments are pushed further back with it.
    let mut new_col = max(max_line + 1, candidates[0].1.unwrap_col());

    if buf[start..=end]
        .iter()
        .all(|(v, _)| matches!(v, Token::Comment(_, _)) || is_newline(v))
    {
        new_col = lines[0];
    }

    for (candidate_idx, (buf_idx, ..)) in candidates.iter().enumerate() {
        match &mut buf[*buf_idx] {
            (Token::Comment(_, pos), _) => {
                *pos = Position::Relative(new_col - lines[candidate_idx])
            }
            _ => unreachable!(),
        }
    }
}

// Consume one line from newline to line-ending comment from `iter` and return
// the line length up to, but not including the comment or its preceding space.
fn line_len<'a, T>(iter: T) -> impl Iterator<Item = usize> + use<'a, T>
where
    T: Iterator<Item = &'a (Token<'a>, Indent)>,
{
    let mut iter = iter.peekable();

    // Invariant: only called with a token iter that will yield a newline as the
    // indentation value is set by the first token after the newline.
    debug_assert!(iter.peek().is_some_and(|v| is_newline(&v.0)));

    let mut len = 0;
    let mut line_tokens = 0;
    std::iter::from_fn(move || {
        loop {
            let (t, _) = iter.next()?;

            // If a newline is observed, clear the accumulated line length and set
            // the indentation level from the first token on the next line (only the
            // first can set the line indentation).
            if is_newline(t) {
                len = iter.peek().unwrap().1.get() as usize * INDENT_STR.len();
                line_tokens = 0;
                continue;
            }

            // If this is the end of line comment, the length has been computed.
            if matches!(t, Token::Comment(..)) && iter.peek().is_some_and(|v| is_newline(&v.0))
                || iter.peek().is_none()
            {
                // If this line contains only a comment, it should not have a -1
                // applied as the only length is the initial indent.
                if line_tokens == 0 {
                    return Some(len);
                }

                // -1 because spaces are inserted before comments or it was a newline.
                return Some(len.saturating_sub(1));
            }

            // Apply the same filtering as when rendering occurs.
            let next = iter.peek();
            if let Some((next, _)) = next {
                if !t.can_precede(next) {
                    continue;
                }
            }

            len += token_len(t);
            line_tokens += 1;

            // Account for any whitespace.
            if let Some(n) = next.map(|(v, _)| t.delimiting_space_len(v)) {
                len += n;
            }
        }
    })
}

#[cfg(test)]
mod tests {
    use crate::assert_rewrite;

    use super::*;

    #[test]
    fn test_line_len_1() {
        let tokens = [
            (Token::Newline, Indent::new(255)),
            // The first token, which sets the indent level
            (Token::And, Indent::new(1)), // 4 + 2 + space
            // The rest, which do not affect indent.
            (Token::Ident("bananas"), Indent::new(255)), // 7 + space
            (Token::Eq, Indent::new(255)),               // 1 + space
            (Token::Lit("42"), Indent::new(255)),        // 2
        ];

        let iter = tokens.iter().peekable();

        let got = line_len(iter).next().unwrap();
        assert_eq!(got, 16);
    }

    #[test]
    fn test_line_len_2() {
        let tokens = [
            (Token::SourceNewline, Indent::new(255)),
            // Immediately followed by second newline
            (Token::Newline, Indent::new(255)),
            // The first token, which sets the indent level
            (Token::And, Indent::new(1)), // 4 + 2 + space
            // The rest, which do not affect indent.
            (Token::Ident("platanos"), Indent::new(255)), // 8
            (Token::Prime, Indent::new(255)),             // 1 + space
            (Token::Eq, Indent::new(255)),                // 1 + space
            (Token::Lit("42"), Indent::new(255)),         // 2
        ];

        let iter = tokens.iter().peekable();

        let got = line_len(iter).next().unwrap();
        assert_eq!(got, 18);
    }

    #[test]
    fn test_line_len_only_comment() {
        let tokens = [
            (Token::SourceNewline, Indent::new(255)),
            (
                Token::Comment("(* bananas *)", Position::Source { row: 2, col: 40 }),
                Indent::new(1),
            ),
            (Token::SourceNewline, Indent::new(255)),
        ];

        let iter = tokens.iter().peekable();

        let got = line_len(iter).next().unwrap();
        assert_eq!(got, 4);
    }

    #[test]
    fn test_comment_manually_aligned() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas = 42       \* This is an important number.
      /\ platanos' = 42     \* That should be assigned here.
====
"
        );
    }

    #[test]
    fn test_comment_partially_aligned() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas = 42         \* This is an important number.
      /\ platanos' = 42       \* That should be assigned here.
      /\ platanos' = 42           \* That should be assigned here.
====
"
        );
    }

    #[test]
    fn test_comment_non_adjacent_partially_aligned() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas = 42       \* This is an important number.
      /\ platanos' = 42         \* That should be assigned here.
      /\ platanos' = 42     \* That should be assigned here.
====
"
        );
    }

    #[test]
    fn test_comment_manually_unaligned() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas = 42      \* This is an important number.
      /\ platanos' = 42     \* That should be assigned here.
====
"
        );
    }

    #[test]
    fn test_comment_align_push() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas = 42    \* This is an important number.
      /\x=4+1+1+1+1+1+1  \* That should be assigned here.
      /\ platanos' = 42  \* That should be assigned here.
====
"
        );
    }

    #[test]
    fn test_comment_align_shrink() {
        assert_rewrite!(
            r"
---- MODULE bananas ----
Op == /\ bananas=42         \* This is an important number.
      /\ platanos'  =  42   \* That should be assigned here.
====
"
        );
    }

    /// Spec fragments with consecutive comment lines, some of which are the
    /// only token on the line (comment only lines).
    mod comment_only_lines {
        use crate::assert_rewrite;

        /// Scenario 1:
        ///
        /// A comment appears in the middle of a block.
        #[test]
        fn test_comment_only_lines_1() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
A ==
    /\ X = 42
    \* Bananas
    \* Platanos
    \* Apples
    /\ Y = 25
====
    "
            );
        }

        /// Scenario 2:
        ///
        /// A comment appears in the middle of a block, and a single
        /// (non-aligned) comment appears at the start of the block.
        #[test]
        fn test_comment_only_lines_2() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
TypeOK ==
    \* Bananas
    /\ X = 42
    \* Platanos
    \* Apples
    /\ Y = 24
====
    "
            );
        }

        /// Scenario 3:
        ///
        /// A comment appears in the middle of a block, and multiple aligned
        /// comments appear at the start of the block.
        #[test]
        fn test_comment_only_lines_3() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
TypeOK ==
    \* Bananas
    \* Bananas
    /\ X = 42
    \* Platanos
    \* Apples
    /\ Y = 24
====
    "
            );
        }

        /// Scenario 4:
        ///
        /// A comment appears at the end of a block.
        #[test]
        fn test_comment_only_lines_4() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
Spec == /\ Init /\ [][Next]_vars
        /\ WF_vars(DetectTermination)
            \* reasonable but not necessary for detecting termination
            \* /\ \A i \in Node : WF_vars(Wakeup(i))
====
    "
            );
        }

        /// Scenario 5:
        ///
        /// One comment only line appears in a block with other comments.
        #[test]
        fn test_comment_only_lines_5() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
Op == /\ bananas = 42       \* This is an important number.
      /\ platanos' = 42     \* Something here.
                            \* Continues here.
      /\ platanos' = 42     \* More here.
====
    "
            );
        }

        /// Scenario 6:
        ///
        /// [test_comment_only_lines_1] with indentation of 3 chars instead of 4.
        #[test]
        fn test_comment_only_lines_6() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
A ==
   /\ X = 42
   \* Bananas
   \* Platanos
   \* Apples
   /\ Y = 25
====
    "
            );
        }

        /// Scenario 7:
        ///
        /// A multi-line box comment starts a block, with indentation of 3
        /// chars.
        #[test]
        fn test_comment_only_lines_7() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
AlwaysResponds ==
  (*************************************************************************)
  (* Some simple liveness properties, implied by the fact that every       *)
  (* request eventually generates a response.                              *)
  (*************************************************************************)
  /\ \A p \in Proc, r \in Reg :
       X = 42
  /\ \A oi \in [proc : Proc, idx : Nat] :
         Y = 24
====
    "
            );
        }

        /// Scenario 8:
        ///
        /// Comments in a position sensitive to indentation limit rewriting.
        #[test]
        fn test_comment_only_lines_8() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
SetToSeqs == UNION {{x \in [1 -> set]:
                            \* A filter applied on each permutation
                            \* generated by [S -> T]
                            Op(x)}}
====
    "
            );
        }

        /// Scenario 9:
        ///
        /// Comments which are inline with statements that are re-indented.
        #[test]
        fn test_comment_only_lines_9() {
            assert_rewrite!(
                r"
---- MODULE bananas ----
TypeOK ==     /\ X = 42
              \* Platanos
              \* Platanos
              /\ Y = 24
====
    "
            )
        }

        /// Scenario 10:
        ///
        /// Comments which are aligned, but part of a connective list that is
        /// rewrote, causing them to no longer be on consecutive lines.
        #[test]
        fn test_comment_only_lines_10() {
            assert_rewrite!(
                r#"
---- MODULE bananas ----
SvrHidenProperty ==
    /\ (\A x \in sTCPLinkSet: /\ x.Type # "Attacker"
                              /\ x.State = "ESTABLISHED") \* C1
    /\ (\A y \in aTCPLinkSet: /\ y.State # "ESTABLISHED") \* C2
====
"#
            );
        }
    }

    /// A test case discovered through fuzzing where the input string contains a
    /// NULL byte, but the parser recovers and emits a sequence of nodes that
    /// have no newline preceding the first comment.
    ///
    /// When scanned backwards there would be no newline found, and the line
    /// length calculation would be fed a buffer that does not start with a
    /// newline causing it to assert.
    #[test]
    fn test_fuzz_input_contains_null() {
        let s = String::from_utf8(vec![
            0x71, 0x00, 0x0a, 0x2a, 0x5c, 0x2a, 0x0a, 0x4a, 0x5c, 0x2a, 0x0a, 0x2b, 0x41, 0x7e,
            0x41,
        ])
        .unwrap();
        assert_rewrite!(&s);
    }
}