logic_tracer/tokens/
mod.rs

1//! This module defines the types of tokens that can be recognized and validated by the
2//! [`Lexer`](crate::components::lexer) and subsequently utilized by the [`Parser`](crate::components::parser).
3//!
4//! It provides a foundational framework for building the lexical components of a language,
5//! including primitive and potentially complex token types.
6//!
7//! ## Usage
8//!
9//! Token types are typically constructed via parsing strings and are extensively used throughout the
10//! lexer and parser components to facilitate language processing.
11
12// Consider re-enabling and documenting the `variables` module if relevant for future extensions.
13// pub mod variables;  // Defines variable-related tokens like identifiers.
14
15pub mod numbers; // Contains definitions for numeric types.
16pub mod operators; // Contains definitions for various operators.
17pub mod variables; // Contains definitions for variable-related tokens.
18
19pub use numbers::*;
20pub use operators::*;
21pub use variables::*; // variables generator (constants, alphabets, hiragana, etc...)
22
23use std::fmt::Debug;
24
25/// Represents a generic token within the language processing system.
26///
27/// All token types must implement this trait to ensure they can be debugged and
28/// dynamically handled via polymorphism. Tokens are generally created from strings and
29/// can be represented back as strings for debugging and logging purposes.
30// * IN COMPUTER SCIENCE.
31// * A token is a string of one or more characters, which are treated as a single unit by a program.
32// * Tokens are the smallest elements of a program, and they are classified by the compiler according to their functionality.
33pub trait Token: Debug {
34    /// Constructs an instance of a token from a string, if possible.
35    fn from_str<S: Into<String>>(string: S) -> Option<Self>
36    where
37        Self: Sized;
38
39    /// Returns a string representation of the token, typically used for debugging.
40    fn to_string(&self) -> String {
41        let mut result = std::any::type_name::<Self>()
42            .split("::")
43            .collect::<Vec<&str>>();
44        result.reverse();
45
46        let mut token_type = result.get(0).unwrap();
47        let token_type = token_type
48            .chars()
49            .next()
50            .unwrap()
51            .to_uppercase()
52            .collect::<String>()
53            + &token_type[1..];
54
55        // format!("{:>12} :: {self:?}", token_type)  // no additional formatting...
56        let token_type = format!("\x1B[3m{}\x1B[0m", token_type); // italic
57        let token_type = format!("\x1B[1m{}\x1B[0m", token_type); // bold
58
59        format!("{token_type:>28} :: {self:?}")
60        // let token_type = format!("\x1B[36m{}\x1B[0m", token_type);  // cyan
61        // let token_type = format!("\x1B[32m{}\x1B[0m", token_type);  // green
62        // format!("{:>36} :: {self:?}", token_type)  // (format with spacing according to the token_type length...)
63        // todo: make this a bit more dynamic...
64    }
65
66}
67
68#[macro_export]
69/// Macro to implement specific token types for a given trait.
70///
71/// This macro simplifies the creation of token structures that implement a common trait,
72/// allowing for polymorphic handling of different token types within the parsing logic.
73macro_rules! impl_enum_token {
74    ($token_type:ident; $trait_name:ident;  //
75        $(
76            $name:ident (
77                $(
78                    $variant:ident => ($($str:expr),+)
79                    $(,)?
80                )+
81            )
82        ),+ $(,)?
83    ) => {
84        crate::impl_token_trait!($token_type; $trait_name; $($name),+);
85        $(
86            #[derive(Debug, Clone, PartialEq)]
87            pub enum $name { $($variant,)+ }
88
89            impl Token for $name {
90                fn from_str<S: Into<String>>(string: S) -> Option<Self> {
91                    match string.into().as_str() {
92                        $($($str)|+ => Some($name::$variant),)+
93                        _ => None
94                    }
95                }
96            }
97
98            impl $trait_name for $name {}  //
99        )+
100    };
101}
102
103#[macro_export]
104/// Macro to implement specific token types for a given trait.
105/// Implements the 'from' method for the token type to create a Box<dyn Trait> from a string.
106///
107/// This macro requires that al token types implement the specified trait and provides a `from_str` method to parse a string into a token.
108macro_rules! impl_token_trait {
109    ($token_type:ident; $trait_name:ident;
110        $(
111            $name:ident  // Name of the token type
112        ),+ $(,)? // Supports multiple token types
113    ) => {
114        impl $token_type {
115            /// Attempts to create a boxed token from a string if it matches any of the specified types.
116            pub fn from<S: Into<String> + Copy>(string: S) -> Option<Box<dyn $trait_name>> {
117                $(if let Some(value) = $name::from_str(string) {
118                    Some(Box::new(value));  // Create some box with the token type
119                })+
120                None  // Return None if no types match
121            }
122        }
123    };
124}
logic_tracer/tokens/mod.rs

logic_tracer/tokens/
mod.rs
