tokit 0.0.0

Blazing fast parser combinators: parse-while-lexing (zero-copy), deterministic LALR-style parsing, no backtracking. Flexible emitters for fail-fast runtime or greedy compiler diagnostics
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

//! Common types for building language-specific ASTs.
//!
//! This module provides generic, reusable building blocks for creating Abstract
//! Syntax Trees (ASTs) across different programming languages. These types are
//! designed to be language-agnostic while maintaining type safety through generic
//! parameters.
//!
//! # Available Types
//!
//! ## Identifiers
//!
//! - [`Ident`]: Generic identifier with span tracking and language marker
//!
//! ## Generic Literal
//!
//! - [`Lit`]: Generic literal (any literal type)
//!
//! ## Numeric Literals
//!
//! - [`LitDecimal`]: Decimal integer (e.g., `42`, `1_000`)
//! - [`LitHex`]: Hexadecimal integer (e.g., `0xFF`, `0x1A2B`)
//! - [`LitOctal`]: Octal integer (e.g., `0o77`, `0o644`)
//! - [`LitBinary`]: Binary integer (e.g., `0b1010`)
//! - [`LitFloat`]: Floating-point (e.g., `3.14`, `1.0e-5`)
//! - [`LitHexFloat`]: Hexadecimal float (e.g., `0x1.8p3`)
//!
//! ## String Literals
//!
//! - [`LitString`]: Single-line string (e.g., `"hello"`)
//! - [`LitMultilineString`]: Multi-line string (e.g., `"""..."""`)
//! - [`LitRawString`]: Raw string (e.g., `r"C:\path"`)
//!
//! ## Character/Byte Literals
//!
//! - [`LitChar`]: Character literal (e.g., `'a'`)
//! - [`LitByte`]: Byte literal (e.g., `b'a'`)
//! - [`LitByteString`]: Byte string (e.g., `b"bytes"`)
//!
//! ## Boolean and Null
//!
//! - [`LitBool`]: Boolean literal (`true`/`false`)
//! - [`LitNull`]: Null/nil literal
//!
//! # Design Principles
//!
//! All types in this module follow these principles:
//!
//! 1. **Generic over string representation**: Support zero-copy (`&str`), owned
//!    (`String`), and interned strings
//! 2. **Span tracking**: Every node carries its source location for diagnostics
//! 3. **Language safety**: Generic `Lang` parameter prevents mixing ASTs from
//!    different languages
//! 4. **Error recovery**: Implement [`ErrorNode`](crate::error::ErrorNode) when
//!    appropriate for creating placeholders during parsing errors
//!
//! # Example: Building a Simple Expression AST
//!
//! ```rust,ignore
//! use tokit::types::Ident;
//! use tokit::utils::Span;
//!
//! // Define your language marker
//! struct MyLang;
//!
//! // Define expression type using Ident
//! enum Expr<'a> {
//!     Variable(Ident<&'a str, MyLang>),
//!     Number(i64, Span),
//!     Add(Box<Expr<'a>>, Box<Expr<'a>>, Span),
//! }
//!
//! // Create an expression
//! let var = Ident::new(Span::new(0, 1), "x");
//! let expr = Expr::Variable(var);
//! ```

use derive_more::{IsVariant, TryUnwrap, Unwrap};

use crate::{
  error::ErrorNode,
  syntax::{Language, Syntax},
  utils::{AsSpan, SimpleSpan},
};

pub use ident::*;
pub use keyword::*;
pub use lit::*;

mod ident;
mod keyword;
mod lit;

/// A type representing a recoverable parse node, which can be a valid node,
/// an error node with span, or a missing node with span.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, IsVariant, TryUnwrap, Unwrap)]
#[unwrap(ref, ref_mut)]
#[try_unwrap(ref, ref_mut)]
pub enum Recoverable<T, S = SimpleSpan> {
  /// A valid parse node.
  Node(T),
  /// An error node with associated span.
  Error(S),
  /// A missing node with associated span.
  Missing(S),
}

impl<T, S> AsSpan<S> for Recoverable<T, S>
where
  T: AsSpan<S>,
{
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn as_span(&self) -> &S {
    match self {
      Self::Node(node) => node.as_span(),
      Self::Error(span) | Self::Missing(span) => span,
    }
  }
}

impl<T, S> Syntax for Recoverable<T, S>
where
  T: Syntax,
{
  type Lang = T::Lang;
  const KIND: <Self::Lang as Language>::SyntaxKind = T::KIND;

  type Component = T::Component;

  type COMPONENTS = T::COMPONENTS;

  type REQUIRED = T::REQUIRED;

  fn possible_components()
  -> &'static generic_arraydeque::GenericArrayDeque<Self::Component, Self::COMPONENTS> {
    T::possible_components()
  }

  fn required_components()
  -> &'static generic_arraydeque::GenericArrayDeque<Self::Component, Self::REQUIRED> {
    T::required_components()
  }
}

impl<T> ErrorNode for Recoverable<T> {
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn error(span: SimpleSpan) -> Self {
    Self::Error(span)
  }

  #[cfg_attr(not(tarpaulin), inline(always))]
  fn missing(span: SimpleSpan) -> Self {
    Self::Missing(span)
  }
}

impl<T> From<T> for Recoverable<T> {
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn from(node: T) -> Self {
    Self::Node(node)
  }
}