vyre-libs 0.6.2

vyre Category A library ecosystem - pure-IR compositions over vyre-ops hardware primitives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

//! GPU lexer plan builder for Rust source text.

mod program;

pub use program::{rust_lexer, rust_lexer_batch};

use vyre::ir::Program;

/// Builder for the GPU Rust lexer.
pub struct RustLexerPlan;

impl RustLexerPlan {
    /// Create a new lexer plan.
    pub fn new() -> Self {
        Self
    }

    /// Build the Vyre IR program for an empty source buffer.
    #[must_use]
    pub fn build(&self) -> Program {
        self.build_for_len(0)
    }

    /// Build the Vyre IR program that lexes a source buffer of `haystack_len`
    /// bytes into compact token columns. The source buffer stores one byte per
    /// `u32` word.
    #[must_use]
    pub fn build_for_len(&self, haystack_len: u32) -> Program {
        rust_lexer(
            "haystack",
            "out_tok_types",
            "out_tok_starts",
            "out_tok_lens",
            "out_counts",
            haystack_len,
        )
    }

    /// Build the Vyre IR program that lexes `source_count` source slices from
    /// a packed source buffer. `token_stride` is the per-source output window
    /// width in token slots.
    #[must_use]
    pub fn build_batch_for_layout(
        &self,
        haystack_len: u32,
        source_count: u32,
        token_stride: u32,
    ) -> Program {
        rust_lexer_batch(
            "haystack",
            "source_offsets",
            "source_lens",
            "out_tok_types",
            "out_tok_starts",
            "out_tok_lens",
            "out_counts",
            haystack_len,
            source_count,
            token_stride,
        )
    }
}

impl Default for RustLexerPlan {
    fn default() -> Self {
        Self::new()
    }
}