rustpython-ruff_python_parser 0.15.8

Unofficial fork for RustPython
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

use ruff_python_ast::token::TokenKind;

/// A bit-set of `TokenKind`s
#[derive(Clone, Copy)]
pub(crate) struct TokenSet(u128);

impl TokenSet {
    pub(crate) const fn new<const N: usize>(kinds: [TokenKind; N]) -> TokenSet {
        let mut res = 0u128;
        let mut i = 0usize;

        while i < N {
            let kind = kinds[i];
            res |= mask(kind);
            i += 1;
        }
        TokenSet(res)
    }

    pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
        TokenSet(self.0 | other.0)
    }

    pub(crate) const fn remove(self, kind: TokenKind) -> TokenSet {
        TokenSet(self.0 & !mask(kind))
    }

    pub(crate) const fn contains(&self, kind: TokenKind) -> bool {
        self.0 & mask(kind) != 0
    }
}

const fn mask(kind: TokenKind) -> u128 {
    1u128 << (kind as usize)
}

impl<const N: usize> From<[TokenKind; N]> for TokenSet {
    fn from(kinds: [TokenKind; N]) -> Self {
        TokenSet::new(kinds)
    }
}

#[test]
fn token_set_works_for_tokens() {
    use ruff_python_ast::token::TokenKind::*;
    let mut ts = TokenSet::new([EndOfFile, Name]);
    assert!(ts.contains(EndOfFile));
    assert!(ts.contains(Name));
    assert!(!ts.contains(Plus));
    ts = ts.remove(Name);
    assert!(!ts.contains(Name));
}