perl-lexer 0.13.1

High-performance Perl lexer with context-aware tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

/// The difference between two [`crate::checkpoint::LexerCheckpoint`]s, produced by
/// [`crate::checkpoint::LexerCheckpoint::diff`].
#[derive(Debug)]
pub struct CheckpointDiff {
    /// Signed byte-offset difference between the two checkpoint positions.
    pub position_delta: isize,
    /// Whether the lexer mode (term vs. operator) changed.
    pub mode_changed: bool,
    /// Whether the nested delimiter stack differs.
    pub delimiter_stack_changed: bool,
    /// Whether any prototype-tracking state differs.
    pub prototype_state_changed: bool,
    /// Whether the [`crate::checkpoint::CheckpointContext`] variant changed.
    pub context_changed: bool,
}

impl CheckpointDiff {
    /// Check if any state changed besides position
    pub fn has_state_changes(&self) -> bool {
        self.mode_changed
            || self.delimiter_stack_changed
            || self.prototype_state_changed
            || self.context_changed
    }
}

#[cfg(test)]
mod tests {
    use super::CheckpointDiff;

    #[test]
    fn has_state_changes_returns_false_when_only_position_differs() {
        let diff = CheckpointDiff {
            position_delta: 7,
            mode_changed: false,
            delimiter_stack_changed: false,
            prototype_state_changed: false,
            context_changed: false,
        };

        assert!(!diff.has_state_changes());
    }

    #[test]
    fn has_state_changes_returns_true_when_mode_changes() {
        let diff = CheckpointDiff {
            position_delta: 0,
            mode_changed: true,
            delimiter_stack_changed: false,
            prototype_state_changed: false,
            context_changed: false,
        };

        assert!(diff.has_state_changes());
    }

    #[test]
    fn has_state_changes_returns_true_when_any_non_position_flag_changes() {
        for diff in [
            CheckpointDiff {
                position_delta: 0,
                mode_changed: false,
                delimiter_stack_changed: true,
                prototype_state_changed: false,
                context_changed: false,
            },
            CheckpointDiff {
                position_delta: 0,
                mode_changed: false,
                delimiter_stack_changed: false,
                prototype_state_changed: true,
                context_changed: false,
            },
            CheckpointDiff {
                position_delta: 0,
                mode_changed: false,
                delimiter_stack_changed: false,
                prototype_state_changed: false,
                context_changed: true,
            },
        ] {
            assert!(diff.has_state_changes());
        }
    }
}