ktrs_parser/builder/
pool.rs

1//! [`PsiBuilder`] construction from per-thread recycled vectors. A file plus its chameleons
2//! create one builder per lazy node; fresh (often multi-megabyte) vectors per builder cost more
3//! in allocation and page faults than parsing a short block does, while recycled ones are warm.
4
5use std::cell::RefCell;
6
7use ktrs_lexer::Token;
8use ktrs_syntax::SyntaxKind;
9
10use super::production::{Production, ProductionVecs};
11use super::psi_builder::PsiBuilder;
12
13/// Enough for any realistic chameleon nesting depth; deeper builders just allocate.
14const MAX_POOLED: usize = 16;
15/// Don't keep buffers of huge files (~650 KB of source and up) alive for the rest of the thread.
16const MAX_POOLED_LEXEMES: usize = 1 << 18;
17
18thread_local! {
19    static POOL: RefCell<Vec<Buffers>> = const { RefCell::new(Vec::new()) };
20}
21
22#[derive(Default)]
23struct Buffers {
24    text: String,
25    lex_starts: Vec<u32>,
26    lex_types: Vec<SyntaxKind>,
27    orig_types: Vec<SyntaxKind>,
28    production: ProductionVecs,
29    skipped_errors: Vec<bool>,
30}
31
32impl Buffers {
33    fn take() -> Buffers {
34        POOL.with(|pool| pool.borrow_mut().pop()).unwrap_or_default()
35    }
36
37    fn give_back(mut self) {
38        if self.lex_types.capacity() > MAX_POOLED_LEXEMES || self.production.markers.capacity() > 4 * MAX_POOLED_LEXEMES {
39            return;
40        }
41        self.text.clear();
42        self.lex_starts.clear();
43        self.lex_types.clear();
44        self.orig_types.clear();
45        self.production.clear();
46        self.skipped_errors.clear();
47        POOL.with(|pool| {
48            let mut pool = pool.borrow_mut();
49            if pool.len() < MAX_POOLED {
50                pool.push(self);
51            }
52        });
53    }
54}
55
56impl PsiBuilder {
57    /// `tokens` must tile `text` exactly (lengths sum to `text.len()`).
58    pub fn new(text: &str, tokens: &[Token]) -> PsiBuilder {
59        PsiBuilder::from_tokens(text, tokens.iter().copied())
60    }
61
62    /// A builder over `text` lexed by the Kotlin lexer.
63    pub(crate) fn lex_kotlin(text: &str) -> PsiBuilder {
64        PsiBuilder::from_tokens(text, ktrs_lexer::tokens_of(text))
65    }
66
67    fn from_tokens(text: &str, tokens: impl Iterator<Item = Token>) -> PsiBuilder {
68        let mut buffers = Buffers::take();
69        let mut offset = 0u32;
70        for token in tokens {
71            buffers.lex_starts.push(offset);
72            buffers.lex_types.push(token.kind);
73            offset += token.len;
74        }
75        buffers.lex_starts.push(offset);
76        PsiBuilder::from_buffers(text, buffers)
77    }
78
79    /// A builder over lexemes cut from another builder: `starts` has one more entry than
80    /// `kinds` and is rebased so that `starts[0]` becomes offset 0 of `text`.
81    pub(crate) fn from_lexemes(text: &str, starts: &[u32], kinds: &[SyntaxKind]) -> PsiBuilder {
82        let mut buffers = Buffers::take();
83        let base = starts[0];
84        buffers.lex_starts.extend(starts.iter().map(|s| s - base));
85        buffers.lex_types.extend_from_slice(kinds);
86        PsiBuilder::from_buffers(text, buffers)
87    }
88
89    fn from_buffers(text: &str, buffers: Buffers) -> PsiBuilder {
90        let Buffers { text: mut own_text, lex_starts, lex_types, mut orig_types, production, skipped_errors } = buffers;
91        assert_eq!(lex_starts.last().copied(), Some(text.len() as u32), "token lengths must sum to the text length");
92        own_text.push_str(text);
93        orig_types.extend_from_slice(&lex_types);
94        PsiBuilder {
95            text: own_text,
96            lex_starts,
97            lex_types,
98            orig_types,
99            current_lexeme: 0,
100            token_type_checked: false,
101            production: Production::from_vecs(production),
102            skipped_errors,
103        }
104    }
105}
106
107impl Drop for PsiBuilder {
108    fn drop(&mut self) {
109        let production = self.production.take_vecs();
110        Buffers {
111            text: std::mem::take(&mut self.text),
112            lex_starts: std::mem::take(&mut self.lex_starts),
113            lex_types: std::mem::take(&mut self.lex_types),
114            orig_types: std::mem::take(&mut self.orig_types),
115            production,
116            skipped_errors: std::mem::take(&mut self.skipped_errors),
117        }
118        .give_back();
119    }
120}
ktrs_parser/builder/pool.rs

ktrs_parser/builder/
pool.rs
