ra_ap_parser/
input.rs

1//! See [`Input`].
2
3use crate::SyntaxKind;
4
5#[allow(non_camel_case_types)]
6type bits = u64;
7
8/// Input for the parser -- a sequence of tokens.
9///
10/// As of now, parser doesn't have access to the *text* of the tokens, and makes
11/// decisions based solely on their classification. Unlike `LexerToken`, the
12/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
13///
14/// Struct of arrays internally, but this shouldn't really matter.
15pub struct Input {
16    kind: Vec<SyntaxKind>,
17    joint: Vec<bits>,
18    contextual_kind: Vec<SyntaxKind>,
19}
20
21/// `pub` impl used by callers to create `Tokens`.
22impl Input {
23    #[inline]
24    pub fn with_capacity(capacity: usize) -> Self {
25        Self {
26            kind: Vec::with_capacity(capacity),
27            joint: Vec::with_capacity(capacity / size_of::<bits>()),
28            contextual_kind: Vec::with_capacity(capacity),
29        }
30    }
31    #[inline]
32    pub fn push(&mut self, kind: SyntaxKind) {
33        self.push_impl(kind, SyntaxKind::EOF)
34    }
35    #[inline]
36    pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
37        self.push_impl(SyntaxKind::IDENT, contextual_kind)
38    }
39    /// Sets jointness for the last token we've pushed.
40    ///
41    /// This is a separate API rather than an argument to the `push` to make it
42    /// convenient both for textual and mbe tokens. With text, you know whether
43    /// the *previous* token was joint, with mbe, you know whether the *current*
44    /// one is joint. This API allows for styles of usage:
45    ///
46    /// ```ignore
47    /// // In text:
48    /// tokens.was_joint(prev_joint);
49    /// tokens.push(curr);
50    ///
51    /// // In MBE:
52    /// token.push(curr);
53    /// tokens.push(curr_joint)
54    /// ```
55    #[inline]
56    pub fn was_joint(&mut self) {
57        let n = self.len() - 1;
58        let (idx, b_idx) = self.bit_index(n);
59        self.joint[idx] |= 1 << b_idx;
60    }
61    #[inline]
62    fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
63        let idx = self.len();
64        if idx.is_multiple_of(bits::BITS as usize) {
65            self.joint.push(0);
66        }
67        self.kind.push(kind);
68        self.contextual_kind.push(contextual_kind);
69    }
70}
71
72/// pub(crate) impl used by the parser to consume `Tokens`.
73impl Input {
74    pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
75        self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
76    }
77    pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
78        self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
79    }
80    pub(crate) fn is_joint(&self, n: usize) -> bool {
81        let (idx, b_idx) = self.bit_index(n);
82        self.joint[idx] & (1 << b_idx) != 0
83    }
84}
85
86impl Input {
87    fn bit_index(&self, n: usize) -> (usize, usize) {
88        let idx = n / (bits::BITS as usize);
89        let b_idx = n % (bits::BITS as usize);
90        (idx, b_idx)
91    }
92    fn len(&self) -> usize {
93        self.kind.len()
94    }
95}
ra_ap_parser/input.rs

ra_ap_parser/
input.rs
