vyre-libs 0.6.3

vyre Category A library ecosystem - pure-IR compositions over vyre-ops hardware primitives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//! Content-keyed Python 3.12 pipeline Program cache.

use std::sync::Arc;

use crate::parsing::python::lex::python312_lexer;
use crate::parsing::python::parse::calls::python312_extract_calls;
use crate::parsing::python::parse::structure::{
    python312_extract_imports, python312_extract_structure,
};
use crate::parsing::source_cache::{source_len_u32_nonzero, ParsedSourceLru};
use vyre::ir::Program;

/// Cached Python pipeline stage bundle for one source shape.
#[derive(Debug, Clone)]
pub struct Python312PipelinePrograms {
    /// Python lexer Program.
    pub lex: Program,
    /// Definition / class / span structural extractor.
    pub structure: Program,
    /// Import extractor.
    pub imports: Program,
    /// Call-site extractor.
    pub calls: Program,
}

/// Bounded content-keyed Python pipeline cache.
pub type Python312PipelineCache = ParsedSourceLru<Python312PipelinePrograms>;

/// Build or fetch the Python 3.12 pipeline bundle for `source`.
pub fn get_or_build_python312_pipeline(
    cache: &Python312PipelineCache,
    source: &[u8],
    extra: &[u8],
) -> Arc<Python312PipelinePrograms> {
    cache.get_or_parse(source, extra, |bytes| {
        let len = source_len_u32_nonzero(bytes);
        Python312PipelinePrograms {
            lex: python312_lexer(
                "haystack",
                "tok_types",
                "tok_starts",
                "tok_lens",
                "tok_counts",
                len,
            ),
            structure: python312_extract_structure(
                "tok_types",
                "tok_starts",
                "tok_lens",
                "structure_records",
                "structure_counts",
                len,
            ),
            imports: python312_extract_imports(
                "tok_types",
                "tok_starts",
                "tok_lens",
                "import_records",
                "import_counts",
                len,
            ),
            calls: python312_extract_calls(
                "tok_types",
                "tok_starts",
                "tok_lens",
                "call_records",
                "call_counts",
                "kwarg_records",
                "kwarg_counts",
                len,
            ),
        }
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn python_pipeline_cache_reuses_same_source_bundle() {
        let cache = Python312PipelineCache::with_capacity(4);
        let a = get_or_build_python312_pipeline(&cache, b"def f():\n    return 1\n", b"py312");
        let b = get_or_build_python312_pipeline(&cache, b"def f():\n    return 1\n", b"py312");
        assert!(Arc::ptr_eq(&a, &b));
        assert_eq!(cache.len(), 1);
    }

    #[test]
    fn python_pipeline_cache_separates_options() {
        let cache = Python312PipelineCache::with_capacity(4);
        let a = get_or_build_python312_pipeline(&cache, b"x = 1\n", b"mode=a");
        let b = get_or_build_python312_pipeline(&cache, b"x = 1\n", b"mode=b");
        assert!(!Arc::ptr_eq(&a, &b));
        assert_eq!(cache.len(), 2);
    }
}