ktrs_parser/builder/
pool.rs

1//! [`PsiBuilder`] construction from per-thread recycled vectors. A file plus its chameleons
2//! create one builder per lazy node; fresh (often multi-megabyte) vectors per builder cost more
3//! in allocation and page faults than parsing a short block does, while recycled ones are warm.
4
5use std::cell::RefCell;
6
7use ktrs_lexer::Token;
8use ktrs_syntax::SyntaxKind;
9
10use super::production::{MarkerData, Production};
11use super::psi_builder::PsiBuilder;
12
13/// Enough for any realistic chameleon nesting depth; deeper builders just allocate.
14const MAX_POOLED: usize = 16;
15/// Don't keep buffers of huge files (~650 KB of source and up) alive for the rest of the thread.
16const MAX_POOLED_LEXEMES: usize = 1 << 18;
17
18thread_local! {
19    static POOL: RefCell<Vec<Buffers>> = const { RefCell::new(Vec::new()) };
20}
21
22#[derive(Default)]
23struct Buffers {
24    text: String,
25    lex_starts: Vec<u32>,
26    lex_types: Vec<SyntaxKind>,
27    orig_types: Vec<SyntaxKind>,
28    markers: Vec<MarkerData>,
29    list: Vec<i32>,
30    skipped_errors: Vec<bool>,
31}
32
33impl Buffers {
34    fn take() -> Buffers {
35        POOL.with(|pool| pool.borrow_mut().pop()).unwrap_or_default()
36    }
37
38    fn give_back(mut self) {
39        if self.lex_types.capacity() > MAX_POOLED_LEXEMES || self.markers.capacity() > 4 * MAX_POOLED_LEXEMES {
40            return;
41        }
42        self.text.clear();
43        self.lex_starts.clear();
44        self.lex_types.clear();
45        self.orig_types.clear();
46        self.markers.clear();
47        self.list.clear();
48        self.skipped_errors.clear();
49        POOL.with(|pool| {
50            let mut pool = pool.borrow_mut();
51            if pool.len() < MAX_POOLED {
52                pool.push(self);
53            }
54        });
55    }
56}
57
58impl PsiBuilder {
59    /// `tokens` must tile `text` exactly (lengths sum to `text.len()`).
60    pub fn new(text: &str, tokens: &[Token]) -> PsiBuilder {
61        PsiBuilder::from_tokens(text, tokens.iter().copied())
62    }
63
64    /// A builder over `text` lexed by the Kotlin lexer.
65    pub(crate) fn lex_kotlin(text: &str) -> PsiBuilder {
66        PsiBuilder::from_tokens(text, ktrs_lexer::tokens_of(text))
67    }
68
69    fn from_tokens(text: &str, tokens: impl Iterator<Item = Token>) -> PsiBuilder {
70        let mut buffers = Buffers::take();
71        let mut offset = 0u32;
72        for token in tokens {
73            buffers.lex_starts.push(offset);
74            buffers.lex_types.push(token.kind);
75            offset += token.len;
76        }
77        buffers.lex_starts.push(offset);
78        PsiBuilder::from_buffers(text, buffers)
79    }
80
81    /// A builder over lexemes cut from another builder: `starts` has one more entry than
82    /// `kinds` and is rebased so that `starts[0]` becomes offset 0 of `text`.
83    pub(crate) fn from_lexemes(text: &str, starts: &[u32], kinds: &[SyntaxKind]) -> PsiBuilder {
84        let mut buffers = Buffers::take();
85        let base = starts[0];
86        buffers.lex_starts.extend(starts.iter().map(|s| s - base));
87        buffers.lex_types.extend_from_slice(kinds);
88        PsiBuilder::from_buffers(text, buffers)
89    }
90
91    fn from_buffers(text: &str, buffers: Buffers) -> PsiBuilder {
92        let Buffers { text: mut own_text, lex_starts, lex_types, mut orig_types, markers, list, skipped_errors } =
93            buffers;
94        assert_eq!(lex_starts.last().copied(), Some(text.len() as u32), "token lengths must sum to the text length");
95        own_text.push_str(text);
96        orig_types.extend_from_slice(&lex_types);
97        PsiBuilder {
98            text: own_text,
99            lex_starts,
100            lex_types,
101            orig_types,
102            current_lexeme: 0,
103            token_type_checked: false,
104            production: Production::from_vecs(markers, list),
105            skipped_errors,
106        }
107    }
108}
109
110impl Drop for PsiBuilder {
111    fn drop(&mut self) {
112        let (markers, list) = self.production.take_vecs();
113        Buffers {
114            text: std::mem::take(&mut self.text),
115            lex_starts: std::mem::take(&mut self.lex_starts),
116            lex_types: std::mem::take(&mut self.lex_types),
117            orig_types: std::mem::take(&mut self.orig_types),
118            markers,
119            list,
120            skipped_errors: std::mem::take(&mut self.skipped_errors),
121        }
122        .give_back();
123    }
124}
ktrs_parser/builder/pool.rs

ktrs_parser/builder/
pool.rs
