mq_lang/
lib.rs

1//! `mq-lang` provides a parser and evaluator for a [mq](https://github.com/harehare/mq).
2//!
3//! ## Examples
4//!
5//! ```rs
6//! let code = "add(\"world!\")";
7//! let input = vec![mq_lang::Value::Markdown(
8//!   mq_markdown::Markdown::from_str("Hello,").unwrap()
9//! )].into_iter();
10//! let mut engine = mq_lang::DefaultEngine::default();
11//!
12//! assert!(matches!(engine.eval(&code, input).unwrap(), mq_lang::Value::String("Hello,world!".to_string())));
13//!
14//! // Parse code into AST nodes
15//! use mq_lang::{tokenize, LexerOptions, AstParser, Arena};
16//! use std::rc::Shared;
17//! use std::cell::SharedCell;
18//!
19//! let code = "1 + 2";
20//! let token_arena = Shared::new(SharedCell::new(Arena::new()));
21//! let ast = mq_lang::parse(code, token_arena).unwrap();
22//!
23//! assert_eq!(ast.nodes.len(), 1);
24//!
25//! // Parse code into CST nodes
26//! use mq_lang::{tokenize, LexerOptions, CstParser};
27//! use std::sync::Arc;
28//!
29//! let code = "1 + 2";
30//! let (cst_nodes, errors) = mq_lang::parse_recovery(code);
31//!
32//! assert!(!errors.has_errors());
33//! assert!(!cst_nodes.is_empty());
34//! ```
35//!
36//! ## Features
37//!
38//! - `ast-json`: Enables serialization and deserialization of the AST (Abstract Syntax Tree)
39//!   to/from JSON format. This also enables the `Engine::eval_ast` method for direct
40//!   AST execution. When this feature is enabled, `serde` and `serde_json` dependencies
41//!   are included.
42mod arena;
43mod ast;
44#[cfg(feature = "cst")]
45mod cst;
46mod engine;
47mod error;
48mod eval;
49mod ident;
50mod lexer;
51mod macro_expand;
52mod module;
53mod number;
54mod range;
55mod selector;
56
57use lexer::Lexer;
58#[cfg(not(feature = "sync"))]
59use std::cell::RefCell;
60#[cfg(not(feature = "sync"))]
61use std::rc::Rc;
62#[cfg(feature = "sync")]
63use std::sync::Arc;
64#[cfg(feature = "sync")]
65use std::sync::RwLock;
66
67pub use arena::{Arena, ArenaId};
68pub use ast::Program;
69pub use ast::node::Expr as AstExpr;
70pub use ast::node::IdentWithToken;
71pub use ast::node::Literal as AstLiteral;
72pub use ast::node::Node as AstNode;
73pub use ast::node::Params as AstParams;
74pub use ast::node::Pattern as AstPattern;
75pub use ast::parser::Parser as AstParser;
76#[cfg(feature = "ast-json")]
77pub use ast::{ast_from_json, ast_to_json};
78pub use engine::Engine;
79pub use error::Error;
80pub use eval::builtin::{
81    BUILTIN_FUNCTION_DOC, BUILTIN_SELECTOR_DOC, BuiltinFunctionDoc, BuiltinSelectorDoc, INTERNAL_FUNCTION_DOC,
82};
83pub use eval::runtime_value::{RuntimeValue, RuntimeValues};
84pub use ident::Ident;
85pub use lexer::Options as LexerOptions;
86pub use lexer::token::{StringSegment, Token, TokenKind};
87pub use module::{
88    BUILTIN_FILE as BUILTIN_MODULE_FILE, Module, ModuleId, ModuleLoader, error::ModuleError,
89    resolver::LocalFsModuleResolver, resolver::ModuleResolver, resolver::module_name,
90};
91pub use range::{Position, Range};
92pub use selector::{AttrKind, Selector};
93
94pub type DefaultEngine = Engine<LocalFsModuleResolver>;
95pub type DefaultModuleLoader = ModuleLoader<LocalFsModuleResolver>;
96
97#[cfg(feature = "cst")]
98pub use cst::incremental::{IncrementalParser, TextEdit};
99#[cfg(feature = "cst")]
100pub use cst::node::BinaryOp as CstBinaryOp;
101#[cfg(feature = "cst")]
102pub use cst::node::Node as CstNode;
103#[cfg(feature = "cst")]
104pub use cst::node::NodeKind as CstNodeKind;
105#[cfg(feature = "cst")]
106pub use cst::node::Trivia as CstTrivia;
107#[cfg(feature = "cst")]
108pub use cst::node::UnaryOp as CstUnaryOp;
109#[cfg(feature = "cst")]
110pub use cst::parser::ErrorReporter as CstErrorReporter;
111#[cfg(feature = "cst")]
112pub use cst::parser::Parser as CstParser;
113
114#[cfg(feature = "debugger")]
115pub use eval::debugger::{
116    Breakpoint, DebugContext, Debugger, DebuggerAction, DebuggerCommand, DebuggerHandler, Source,
117};
118
119use crate::ast::TokenId;
120
121pub type MqResult = Result<RuntimeValues, Box<Error>>;
122
123/// Type alias for reference-counted pointer, switches between Shared and Arc depending on "sync" feature.
124#[cfg(not(feature = "sync"))]
125pub type Shared<T> = Rc<T>;
126#[cfg(feature = "sync")]
127pub type Shared<T> = Arc<T>;
128
129/// Type alias for interior mutability, switches between SharedCell and RwLock depending on "sync" feature.
130#[cfg(not(feature = "sync"))]
131pub type SharedCell<T> = RefCell<T>;
132#[cfg(feature = "sync")]
133pub type SharedCell<T> = RwLock<T>;
134
135pub(crate) type TokenArena = Shared<SharedCell<Arena<Shared<Token>>>>;
136
137#[cfg(feature = "cst")]
138pub fn parse_recovery(code: &str) -> (Vec<Shared<CstNode>>, CstErrorReporter) {
139    let tokens = Lexer::new(lexer::Options {
140        ignore_errors: true,
141        include_spaces: true,
142    })
143    .tokenize(code, Module::TOP_LEVEL_MODULE_ID)
144    .map_err(|e| Box::new(error::Error::from_error(code, e.into(), DefaultModuleLoader::default())))
145    .unwrap();
146
147    let token_vec: Vec<Shared<Token>> = tokens.into_iter().map(Shared::new).collect();
148    CstParser::new(&token_vec).parse()
149}
150
151pub fn parse(code: &str, token_arena: TokenArena) -> Result<Program, Box<error::Error>> {
152    let tokens = Lexer::new(lexer::Options::default())
153        .tokenize(code, Module::TOP_LEVEL_MODULE_ID)
154        .map_err(|e| Box::new(error::Error::from_error(code, e.into(), DefaultModuleLoader::default())))?;
155    let mut token_arena = {
156        #[cfg(not(feature = "sync"))]
157        {
158            token_arena.borrow_mut()
159        }
160
161        #[cfg(feature = "sync")]
162        {
163            token_arena.write().unwrap()
164        }
165    };
166
167    AstParser::new(
168        tokens.into_iter().map(Shared::new).collect::<Vec<_>>().iter(),
169        &mut token_arena,
170        Module::TOP_LEVEL_MODULE_ID,
171    )
172    .parse()
173    .map_err(|e| Box::new(error::Error::from_error(code, e.into(), DefaultModuleLoader::default())))
174}
175
176/// Parses an MDX string and returns an iterator over `Value` nodes.
177pub fn parse_mdx_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
178    let mdx = mq_markdown::Markdown::from_mdx_str(input)?;
179    Ok(mdx.nodes.into_iter().map(RuntimeValue::from).collect())
180}
181
182pub fn parse_html_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
183    let html = mq_markdown::Markdown::from_html_str(input)?;
184    Ok(html.nodes.into_iter().map(RuntimeValue::from).collect())
185}
186
187pub fn parse_html_input_with_options(
188    input: &str,
189    options: mq_markdown::ConversionOptions,
190) -> miette::Result<Vec<RuntimeValue>> {
191    let html = mq_markdown::Markdown::from_html_str_with_options(input, options)?;
192    Ok(html.nodes.into_iter().map(RuntimeValue::from).collect())
193}
194
195/// Parses a Markdown string and returns an iterator over `Value` nodes.
196pub fn parse_markdown_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
197    let md = mq_markdown::Markdown::from_markdown_str(input)?;
198    Ok(md.nodes.into_iter().map(RuntimeValue::from).collect())
199}
200
201/// Parses a plain text string and returns an iterator over `Value` node.
202pub fn parse_text_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
203    Ok(input.lines().map(|line| line.to_string().into()).collect())
204}
205
206/// Returns a vector containing a single `Value` representing an empty input.
207pub fn null_input() -> Vec<RuntimeValue> {
208    vec!["".to_string().into()]
209}
210
211/// Parses a raw input string and returns a vector containing a single `Value` node.
212pub fn raw_input(input: &str) -> Vec<RuntimeValue> {
213    vec![input.to_string().into()]
214}
215
216#[inline(always)]
217pub(crate) fn token_alloc(arena: &TokenArena, token: &Shared<Token>) -> TokenId {
218    #[cfg(not(feature = "sync"))]
219    {
220        arena.borrow_mut().alloc(Shared::clone(token))
221    }
222
223    #[cfg(feature = "sync")]
224    {
225        arena.write().unwrap().alloc(Shared::clone(token))
226    }
227}
228
229#[inline(always)]
230pub(crate) fn get_token(arena: TokenArena, token_id: TokenId) -> Shared<Token> {
231    #[cfg(not(feature = "sync"))]
232    {
233        Shared::clone(&arena.borrow()[token_id])
234    }
235
236    #[cfg(feature = "sync")]
237    {
238        Shared::clone(&arena.read().unwrap()[token_id])
239    }
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245
246    #[test]
247    fn test_eval_basic() {
248        let code = "add(\"world!\")";
249        let input = mq_markdown::Markdown::from_markdown_str("Hello,").unwrap();
250        let mut engine = DefaultEngine::default();
251
252        assert_eq!(
253            engine
254                .eval(
255                    code,
256                    input
257                        .nodes
258                        .into_iter()
259                        .map(RuntimeValue::from)
260                        .collect::<Vec<_>>()
261                        .into_iter()
262                )
263                .unwrap(),
264            vec![RuntimeValue::Markdown(
265                mq_markdown::Node::Text(mq_markdown::Text {
266                    value: "Hello,world!".to_string(),
267                    position: None
268                },),
269                None
270            )]
271            .into()
272        );
273    }
274
275    #[test]
276    fn test_parse_error_syntax() {
277        let code = "add(1,";
278        let token_arena = Shared::new(SharedCell::new(Arena::new(10)));
279        let result = parse(code, token_arena);
280
281        assert!(result.is_err());
282    }
283
284    #[test]
285    fn test_parse_error_lexer() {
286        let code = "add(1, `unclosed string)";
287        let token_arena = Shared::new(SharedCell::new(Arena::new(10)));
288        let result = parse(code, token_arena);
289
290        assert!(result.is_err());
291    }
292
293    #[test]
294    #[cfg(feature = "cst")]
295    fn test_parse_recovery_success() {
296        let code = "add(1, 2)";
297        let (cst_nodes, errors) = parse_recovery(code);
298
299        assert!(!errors.has_errors());
300        assert!(!cst_nodes.is_empty());
301    }
302
303    #[test]
304    #[cfg(feature = "cst")]
305    fn test_parse_recovery_with_errors() {
306        let code = "add(1,";
307        let (cst_nodes, errors) = parse_recovery(code);
308
309        assert!(errors.has_errors());
310        assert!(cst_nodes.is_empty());
311    }
312
313    #[test]
314    #[cfg(feature = "cst")]
315    fn test_parse_recovery_with_error_lexer() {
316        let code = "add(1, \"";
317        let (cst_nodes, errors) = parse_recovery(code);
318
319        assert!(errors.has_errors());
320        assert!(cst_nodes.is_empty());
321    }
322
323    #[test]
324    fn test_parse_markdown_input() {
325        let input = "# Heading\n\nSome text.";
326        let result = parse_markdown_input(input);
327        assert!(result.is_ok());
328        let values: Vec<RuntimeValue> = result.unwrap();
329        assert!(!values.is_empty());
330    }
331
332    #[test]
333    fn test_parse_mdx_input() {
334        let input = "# Heading\n\nSome text.";
335        let result = parse_mdx_input(input);
336        assert!(result.is_ok());
337        let values: Vec<RuntimeValue> = result.unwrap();
338        assert!(!values.is_empty());
339    }
340
341    #[test]
342    fn test_parse_text_input() {
343        let input = "line1\nline2\nline3";
344        let result = parse_text_input(input);
345        assert!(result.is_ok());
346        let values: Vec<RuntimeValue> = result.unwrap();
347        assert_eq!(values.len(), 3);
348    }
349
350    #[test]
351    fn test_parse_html_input() {
352        let input = "<h1>Heading</h1><p>Some text.</p>";
353        let result = parse_html_input(input);
354        assert!(result.is_ok());
355        let values: Vec<RuntimeValue> = result.unwrap();
356        assert!(!values.is_empty());
357    }
358
359    #[test]
360    fn test_parse_html_input_with_options() {
361        let input = r#"<html>
362      <head>
363        <title>Title</title>
364        <meta name="description" content="This is a test meta description.">
365        <script>let foo = 'bar'</script>
366      </head>
367      <body>
368        <p>Some text.</p>
369      </body>
370    </html>"#;
371        let result = parse_html_input_with_options(
372            input,
373            mq_markdown::ConversionOptions {
374                extract_scripts_as_code_blocks: true,
375                generate_front_matter: true,
376                use_title_as_h1: true,
377            },
378        );
379        assert!(result.is_ok());
380        assert_eq!(
381            mq_markdown::Markdown::new(
382                result
383                    .unwrap()
384                    .iter()
385                    .map(|value| match value {
386                        RuntimeValue::Markdown(node, _) => node.clone(),
387                        _ => value.to_string().into(),
388                    })
389                    .collect()
390            )
391            .to_string(),
392            "---
393description: This is a test meta description.
394title: Title
395---
396
397# Title
398
399```
400let foo = 'bar'
401```
402
403Some text.
404"
405        );
406    }
407}
mq_lang/lib.rs

mq_lang/
lib.rs
