mq_lang/
module.rs

1pub mod error;
2pub mod resolver;
3
4use crate::{
5    Arena, ArenaId, Program, Shared, TokenArena,
6    ast::{node as ast, parser::Parser},
7    lexer::{self, Lexer},
8    module::{
9        error::ModuleError,
10        resolver::{DefaultModuleResolver, ModuleResolver},
11    },
12};
13use rustc_hash::FxHashMap;
14use smol_str::SmolStr;
15use std::{borrow::Cow, cell::RefCell, path::PathBuf, sync::LazyLock};
16
17use crate::Token;
18
19struct BuiltinCache {
20    tokens: Vec<Shared<Token>>,
21    module: Module,
22}
23
24thread_local! {
25    static BUILTIN_CACHE: RefCell<Option<BuiltinCache>> = const { RefCell::new(None) };
26}
27
28pub type ModuleId = ArenaId<ModuleName>;
29
30type ModuleName = SmolStr;
31type StandardModules = FxHashMap<SmolStr, fn() -> &'static str>;
32
33impl<T: ModuleResolver> Default for ModuleLoader<T> {
34    fn default() -> Self {
35        Self::new(T::default())
36    }
37}
38
39fn get_module_name(name: &str) -> Cow<'static, str> {
40    // For common module names, use static strings to avoid allocation
41    match name {
42        "ast" => Cow::Borrowed("ast.mq"),
43        "cbor" => Cow::Borrowed("cbor.mq"),
44        "csv" => Cow::Borrowed("csv.mq"),
45        "fuzzy" => Cow::Borrowed("fuzzy.mq"),
46        "hcl" => Cow::Borrowed("hcl.mq"),
47        "json" => Cow::Borrowed("json.mq"),
48        "section" => Cow::Borrowed("section.mq"),
49        "semver" => Cow::Borrowed("semver.mq"),
50        "test" => Cow::Borrowed("test.mq"),
51        "table" => Cow::Borrowed("table.mq"),
52        "toml" => Cow::Borrowed("toml.mq"),
53        "toon" => Cow::Borrowed("toon.mq"),
54        "xml" => Cow::Borrowed("xml.mq"),
55        "yaml" => Cow::Borrowed("yaml.mq"),
56        _ => Cow::Owned(format!("{}.mq", name)),
57    }
58}
59
60#[derive(Debug, Clone)]
61pub struct ModuleLoader<T: ModuleResolver = DefaultModuleResolver> {
62    pub(crate) loaded_modules: Arena<ModuleName>,
63    #[cfg(feature = "debugger")]
64    pub(crate) source_code: Option<String>,
65    source_cache: FxHashMap<SmolStr, String>,
66    resolver: T,
67    /// Tracks sub-module loading depth; HTTP imports are blocked when this is greater than zero.
68    #[cfg(feature = "http-import")]
69    http_depth: usize,
70}
71
72#[derive(Debug, Clone, PartialEq)]
73pub struct Module {
74    pub name: String,
75    pub functions: Program,
76    pub modules: Program,
77    pub vars: Program,
78    pub macros: Program,
79}
80
81impl Module {
82    pub const BUILTIN_MODULE: &str = "builtin";
83    pub const TOP_LEVEL_MODULE: &str = "top-level";
84    pub const TOP_LEVEL_MODULE_ID: ArenaId<ModuleName> = ArenaId::new(0);
85}
86
87pub static STANDARD_MODULES: LazyLock<StandardModules> = LazyLock::new(|| {
88    let mut map = FxHashMap::default();
89
90    macro_rules! std_module {
91        ($name:ident) => {
92            fn $name() -> &'static str {
93                include_str!(concat!("../modules/", stringify!($name), ".mq"))
94            }
95            map.insert(SmolStr::new(stringify!($name)), $name as fn() -> &'static str);
96        };
97    }
98
99    std_module!(ast);
100    std_module!(cbor);
101    std_module!(csv);
102    std_module!(fuzzy);
103    std_module!(hcl);
104    std_module!(json);
105    std_module!(section);
106    std_module!(semver);
107    std_module!(test);
108    std_module!(table);
109    std_module!(toml);
110    std_module!(toon);
111    std_module!(xml);
112    std_module!(yaml);
113
114    map
115});
116
117pub const BUILTIN_FILE: &str = include_str!("../builtin.mq");
118
119impl<T: ModuleResolver> ModuleLoader<T> {
120    pub fn new(resolver: T) -> Self {
121        let mut loaded_modules = Arena::new(10);
122        loaded_modules.alloc(Module::TOP_LEVEL_MODULE.into());
123
124        Self {
125            loaded_modules,
126            #[cfg(feature = "debugger")]
127            source_code: None,
128            source_cache: FxHashMap::default(),
129            resolver,
130            #[cfg(feature = "http-import")]
131            http_depth: 0,
132        }
133    }
134
135    #[inline(always)]
136    pub fn module_name(&self, module_id: ModuleId) -> Cow<'static, str> {
137        match module_id {
138            Module::TOP_LEVEL_MODULE_ID => Cow::Borrowed(Module::TOP_LEVEL_MODULE),
139            _ => self
140                .loaded_modules
141                .get(module_id)
142                .map(|s| Cow::Owned(s.to_string()))
143                .unwrap_or_else(|| Cow::Borrowed("<unknown>")),
144        }
145    }
146
147    pub fn get_module_path(&self, module_name: &str) -> Result<String, ModuleError> {
148        self.resolver.get_path(module_name)
149    }
150
151    #[cfg(feature = "debugger")]
152    pub fn set_source_code(&mut self, source_code: String) {
153        self.source_code = Some(source_code);
154    }
155
156    pub fn search_paths(&self) -> Vec<PathBuf> {
157        self.resolver.search_paths()
158    }
159
160    pub fn set_search_paths(&mut self, paths: Vec<PathBuf>) {
161        self.resolver.set_search_paths(paths);
162    }
163
164    pub fn load(&mut self, module_name: &str, code: &str, token_arena: TokenArena) -> Result<Module, ModuleError> {
165        if self.loaded_modules.contains(module_name.into()) {
166            return Err(ModuleError::AlreadyLoaded(Cow::Owned(module_name.to_string())));
167        }
168
169        let module_id = self.loaded_modules.len().into();
170        let mut program = Self::parse_program(code, module_id, token_arena)?;
171
172        self.load_from_ast(module_name, &mut program)
173    }
174
175    pub fn load_from_ast(&mut self, module_name: &str, program: &mut Program) -> Result<Module, ModuleError> {
176        if self.loaded_modules.contains(module_name.into()) {
177            return Err(ModuleError::AlreadyLoaded(Cow::Owned(module_name.to_string())));
178        }
179
180        let modules = program
181            .iter()
182            .filter(|node| {
183                matches!(
184                    *node.expr,
185                    ast::Expr::Include(_) | ast::Expr::Module(_, _) | ast::Expr::Import(_)
186                )
187            })
188            .cloned()
189            .collect::<Vec<_>>();
190
191        let functions = program
192            .iter()
193            .filter(|node| matches!(*node.expr, ast::Expr::Def(..)))
194            .cloned()
195            .collect::<Vec<_>>();
196
197        let vars = program
198            .iter()
199            .filter(|node| matches!(*node.expr, ast::Expr::Let(..)))
200            .cloned()
201            .collect::<Vec<_>>();
202
203        let macros = program
204            .iter()
205            .filter(|node| matches!(*node.expr, ast::Expr::Macro(..)))
206            .cloned()
207            .collect::<Vec<_>>();
208
209        let expected_len = functions.len() + modules.len() + vars.len() + macros.len();
210
211        if program.len() != expected_len {
212            return Err(ModuleError::InvalidModule);
213        }
214
215        self.loaded_modules.alloc(module_name.into());
216
217        Ok(Module {
218            name: module_name.to_string(),
219            functions,
220            modules,
221            vars,
222            macros,
223        })
224    }
225
226    pub fn canonical_name<'a>(&self, module_path: &'a str) -> &'a str {
227        self.resolver.canonical_name(module_path)
228    }
229
230    pub fn load_from_file(&mut self, module_path: &str, token_arena: TokenArena) -> Result<Module, ModuleError> {
231        // Check before resolving to avoid unnecessary I/O (disk read or network fetch)
232        // when the same module is imported more than once.
233        let name = self.resolver.canonical_name(module_path).to_owned();
234        if self.loaded_modules.contains(name.as_str().into()) {
235            return Err(ModuleError::AlreadyLoaded(Cow::Owned(name)));
236        }
237        let program = self.resolve(module_path)?;
238        self.source_cache.insert(SmolStr::new(&name), program.clone());
239        self.load(&name, &program, token_arena)
240    }
241
242    pub fn resolve(&self, module_name: &str) -> Result<String, ModuleError> {
243        #[cfg(feature = "http-import")]
244        if self.http_depth > 0
245            && (resolver::http_resolver::HttpModuleResolver::is_remote_url(module_name)
246                || resolver::http_resolver::HttpModuleResolver::is_github_url(module_name))
247        {
248            return Err(ModuleError::HttpImportNotAllowed(std::borrow::Cow::Owned(
249                module_name.to_string(),
250            )));
251        }
252        self.resolver.resolve(module_name)
253    }
254
255    /// Signals that sub-module loading has begun; HTTP imports are blocked while depth > 0.
256    #[cfg(feature = "http-import")]
257    pub fn push_http_boundary(&mut self) {
258        self.http_depth = self.http_depth.saturating_add(1);
259    }
260
261    /// Signals that sub-module loading has ended.
262    #[cfg(feature = "http-import")]
263    pub fn pop_http_boundary(&mut self) {
264        self.http_depth = self.http_depth.saturating_sub(1);
265    }
266
267    pub fn load_builtin(&mut self, token_arena: TokenArena) -> Result<Module, ModuleError> {
268        if self.loaded_modules.contains(Module::BUILTIN_MODULE.into()) {
269            return Err(ModuleError::AlreadyLoaded(Cow::Borrowed(Module::BUILTIN_MODULE)));
270        }
271
272        // Cache is only valid when both arenas are in their initial state (builtin
273        // module_id == 1, tokens right after the dummy EOF). Fall back to full parse otherwise.
274        let pristine = self.loaded_modules.len() == 1 && {
275            #[cfg(not(feature = "sync"))]
276            {
277                token_arena.borrow().len() == 1
278            }
279            #[cfg(feature = "sync")]
280            {
281                token_arena.read().unwrap().len() == 1
282            }
283        };
284
285        if pristine {
286            let cached =
287                BUILTIN_CACHE.with(|cache| cache.borrow().as_ref().map(|c| (c.tokens.clone(), c.module.clone())));
288
289            if let Some((tokens, module)) = cached {
290                {
291                    #[cfg(not(feature = "sync"))]
292                    token_arena.borrow_mut().extend_from_slice(&tokens);
293                    #[cfg(feature = "sync")]
294                    token_arena.write().unwrap().extend_from_slice(&tokens);
295                }
296                self.loaded_modules.alloc(Module::BUILTIN_MODULE.into());
297                return Ok(module);
298            }
299        }
300
301        let module = self.load(Module::BUILTIN_MODULE, BUILTIN_FILE, Shared::clone(&token_arena))?;
302
303        if pristine {
304            let tokens = {
305                #[cfg(not(feature = "sync"))]
306                let arena = token_arena.borrow();
307                #[cfg(feature = "sync")]
308                let arena = token_arena.read().unwrap();
309                arena.as_slice()[1..].iter().map(Shared::clone).collect::<Vec<_>>()
310            };
311
312            BUILTIN_CACHE.with(|cache| {
313                *cache.borrow_mut() = Some(BuiltinCache {
314                    tokens,
315                    module: module.clone(),
316                });
317            });
318        }
319
320        Ok(module)
321    }
322
323    #[cfg(feature = "debugger")]
324    pub fn get_source_code_for_debug(&self, module_id: ModuleId) -> Result<String, ModuleError> {
325        let name = self.module_name(module_id);
326        match name.as_ref() {
327            Module::TOP_LEVEL_MODULE => Ok(self.source_code.clone().unwrap_or_default()),
328            Module::BUILTIN_MODULE => Ok(BUILTIN_FILE.to_string()),
329            module_name => {
330                if let Some(cached) = self.source_cache.get(module_name) {
331                    return Ok(cached.clone());
332                }
333                self.resolve(module_name)
334            }
335        }
336    }
337
338    pub fn get_source_code(&self, module_id: ModuleId, source_code: String) -> Result<String, ModuleError> {
339        let name = self.module_name(module_id);
340        match name.as_ref() {
341            Module::TOP_LEVEL_MODULE => Ok(source_code),
342            Module::BUILTIN_MODULE => Ok(BUILTIN_FILE.to_string()),
343            module_name => {
344                if let Some(cached) = self.source_cache.get(module_name) {
345                    return Ok(cached.clone());
346                }
347                self.resolve(module_name)
348            }
349        }
350    }
351
352    /// Returns the display filename for a module (e.g. `"builtin.mq"`, `"csv.mq"`, `""` for top-level).
353    pub fn module_file_name(&self, module_id: ModuleId) -> String {
354        let name = self.module_name(module_id);
355        match name.as_ref() {
356            Module::TOP_LEVEL_MODULE => String::new(),
357            other => get_module_name(other).to_string(),
358        }
359    }
360
361    fn parse_program(code: &str, module_id: ModuleId, token_arena: TokenArena) -> Result<Program, ModuleError> {
362        let tokens = Lexer::new(lexer::Options::default()).tokenize(code, module_id)?;
363        let mut token_arena = {
364            #[cfg(not(feature = "sync"))]
365            {
366                token_arena.borrow_mut()
367            }
368
369            #[cfg(feature = "sync")]
370            {
371                token_arena.write().unwrap()
372            }
373        };
374
375        let program = Parser::new(
376            tokens.into_iter().map(Shared::new).collect::<Vec<_>>().iter(),
377            &mut token_arena,
378            module_id,
379        )
380        .parse()?;
381
382        Ok(program)
383    }
384}
385
386#[cfg(feature = "http-import")]
387impl ModuleLoader<DefaultModuleResolver> {
388    /// Replaces the HTTP resolver's domain allowlist.
389    pub fn set_http_allowed_domains(&mut self, domains: Vec<String>) {
390        self.resolver.set_allowed_domains(domains);
391    }
392
393    /// Clears all locally-cached HTTP module files.
394    ///
395    /// Call this once before processing to force a re-fetch of all cached modules.
396    pub fn clear_http_cache(&self) -> Result<(), error::ModuleError> {
397        self.resolver.clear_http_cache()
398    }
399
400    /// Clears all HTTP module cache including versioned modules and lock files.
401    pub fn clear_http_cache_all(&self) -> Result<(), error::ModuleError> {
402        self.resolver.clear_http_cache_all()
403    }
404}
405
406#[cfg(test)]
407mod tests {
408    use rstest::{fixture, rstest};
409    use smallvec::{SmallVec, smallvec};
410    use smol_str::SmolStr;
411
412    use crate::{
413        Range, Shared, SharedCell, Token, TokenKind,
414        ast::node::{self as ast, IdentWithToken, Param},
415        module::resolver::DefaultModuleResolver,
416        range::Position,
417        token_alloc,
418    };
419
420    use super::{Module, ModuleError, ModuleLoader};
421
422    #[fixture]
423    fn token_arena() -> Shared<SharedCell<crate::arena::Arena<Shared<Token>>>> {
424        Shared::new(SharedCell::new(crate::arena::Arena::new(10)))
425    }
426
427    /// Arena that mirrors the engine's initial state: one dummy EOF token at index 0.
428    /// Required to exercise the "pristine" cache path in `load_builtin`.
429    #[fixture]
430    fn pristine_token_arena() -> Shared<SharedCell<crate::arena::Arena<Shared<Token>>>> {
431        let arena = Shared::new(SharedCell::new(crate::arena::Arena::new(2048)));
432        token_alloc(
433            &arena,
434            &Shared::new(Token {
435                kind: TokenKind::Eof,
436                range: Range::default(),
437                module_id: Module::TOP_LEVEL_MODULE_ID,
438            }),
439        );
440        arena
441    }
442
443    #[rstest]
444    #[case::load1("test".to_string(), Err(ModuleError::InvalidModule))]
445    #[case::load2("let test = \"value\"".to_string(), Ok(Module{
446        name: "test".to_string(),
447        functions: Vec::new(),
448        modules: Vec::new(),
449        vars: vec![
450            Shared::new(ast::Node{token_id: 0.into(), expr: Shared::new(ast::Expr::Let(
451                ast::Pattern::Ident(IdentWithToken::new_with_token("test", Some(Shared::new(Token{
452                    kind: TokenKind::Ident(SmolStr::new("test")),
453                    range: Range{start: Position{line: 1, column: 5}, end: Position{line: 1, column: 9}},
454                    module_id: 1.into()
455                })))),
456                Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::String("value".to_string())))})
457            ))})],
458        macros: Vec::new(),
459    }))]
460    #[case::load3("def test(): 1;".to_string(), Ok(Module{
461        name: "test".to_string(),
462        modules: Vec::new(),
463        functions: vec![
464            Shared::new(ast::Node{token_id: 0.into(), expr: Shared::new(ast::Expr::Def(
465            IdentWithToken::new_with_token("test", Some(Shared::new(Token{
466                kind: TokenKind::Ident(SmolStr::new("test")),
467                range: Range{start: Position{line: 1, column: 5}, end: Position{line: 1, column: 9}},
468                module_id: 1.into()
469            }))),
470            SmallVec::new(),
471            vec![
472                Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::Number(1.into())))})
473            ]
474            ))})],
475        vars: Vec::new(),
476        macros: Vec::new(),
477    }))]
478    #[case::load4("def test(a, b): add(a, b);".to_string(), Ok(Module{
479        name: "test".to_string(),
480        modules: Vec::new(),
481        functions: vec![
482            Shared::new(ast::Node{token_id: 0.into(), expr: Shared::new(ast::Expr::Def(
483                IdentWithToken::new_with_token("test", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("test")), range: Range{start: Position{line: 1, column: 5}, end: Position{line: 1, column: 9}}, module_id: 1.into()}))),
484                smallvec![
485                    Param::new(IdentWithToken::new_with_token("a", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("a")), range: Range{start: Position{line: 1, column: 10}, end: Position{line: 1, column: 11}}, module_id: 1.into()})))),
486                    Param::new(IdentWithToken::new_with_token("b", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("b")), range: Range{start: Position{line: 1, column: 13}, end: Position{line: 1, column: 14}}, module_id: 1.into()})))),
487                ],
488                vec![
489                    Shared::new(ast::Node{token_id: 4.into(), expr: Shared::new(ast::Expr::Call(
490                    IdentWithToken::new_with_token("add", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("add")), range: Range{start: Position{line: 1, column: 17}, end: Position{line: 1, column: 20}}, module_id: 1.into()}))),
491                    smallvec![
492                        Shared::new(ast::Node{token_id: 2.into(),
493                            expr: Shared::new(
494                                ast::Expr::Ident(IdentWithToken::new_with_token("a", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("a")), range: Range{start: Position{line: 1, column: 21}, end: Position{line: 1, column: 22}}, module_id: 1.into()}))))
495                                )}),
496                        Shared::new(ast::Node{token_id: 3.into(),
497                            expr: Shared::new(
498                                ast::Expr::Ident(IdentWithToken::new_with_token("b", Some(Shared::new(Token{kind: TokenKind::Ident(SmolStr::new("b")), range: Range{start: Position{line: 1, column: 24}, end: Position{line: 1, column: 25}}, module_id: 1.into()}))))
499                            )})
500                    ],
501                ))})]
502            ))})],
503        vars: Vec::new(),
504        macros: Vec::new(),
505    }))]
506    fn test_load(
507        token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
508        #[case] program: String,
509        #[case] expected: Result<Module, ModuleError>,
510    ) {
511        assert_eq!(
512            ModuleLoader::new(DefaultModuleResolver::default()).load("test", &program, token_arena),
513            expected
514        );
515    }
516
517    #[rstest]
518    #[case::load_standard_csv("csv", Ok(Module {
519        name: "csv".to_string(),
520        functions: Vec::new(),
521        modules: Vec::new(), // Assuming the csv.mq only contains definitions or is empty for this test
522        vars: Vec::new(),
523        macros: Vec::new(),
524    }))]
525    fn test_load_standard_module(
526        token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
527        #[case] module_name: &str,
528        #[case] expected: Result<Module, ModuleError>,
529    ) {
530        let mut loader = ModuleLoader::new(DefaultModuleResolver::default());
531        let result = loader.load_from_file(module_name, token_arena.clone());
532        // Only check that loading does not return NotFound error and returns Some(Module)
533        match expected {
534            Ok(_) => {
535                assert!(result.is_ok(), "Expected Ok, got {:?}", result);
536                assert_eq!(result.unwrap().name, module_name);
537            }
538            Err(ref e) => {
539                assert_eq!(result.unwrap_err(), *e);
540            }
541        }
542    }
543
544    #[test]
545    fn test_standard_modules_contains_csv() {
546        assert!(super::STANDARD_MODULES.contains_key("csv"));
547        let csv_content = super::STANDARD_MODULES.get("csv").unwrap()();
548        assert!(csv_content.contains("")); // Just check it's a string, optionally check for expected content
549    }
550
551    #[test]
552    fn test_load_builtin_idempotent() {
553        let token_arena = token_arena();
554        let mut loader = ModuleLoader::new(DefaultModuleResolver::default());
555        assert!(loader.load_builtin(Shared::clone(&token_arena)).is_ok());
556        // Second call on the same loader must return AlreadyLoaded, not corrupt state.
557        assert!(matches!(
558            loader.load_builtin(Shared::clone(&token_arena)),
559            Err(ModuleError::AlreadyLoaded(_))
560        ));
561    }
562
563    #[test]
564    fn test_load_builtin_non_pristine_falls_back_to_parse() {
565        // Load another module first so the arenas are no longer in their initial state.
566        let token_arena = token_arena();
567        let mut loader = ModuleLoader::new(DefaultModuleResolver::default());
568        loader
569            .load("other", "def dummy(): 1;", Shared::clone(&token_arena))
570            .expect("should load other module");
571
572        // load_builtin must still succeed even though the arenas are non-pristine.
573        let result = loader.load_builtin(Shared::clone(&token_arena));
574        assert!(result.is_ok(), "load_builtin failed on non-pristine state: {result:?}");
575
576        let module = result.unwrap();
577        assert_eq!(module.name, Module::BUILTIN_MODULE);
578    }
579
580    /// Token arena size must be the same whether the builtin module was loaded from a fresh
581    /// parse or replayed from the thread-local cache.
582    #[rstest]
583    fn test_load_builtin_cache_arena_size_consistent(
584        pristine_token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
585    ) {
586        let arena1 = pristine_token_arena;
587        let mut loader1 = ModuleLoader::new(DefaultModuleResolver::default());
588        loader1.load_builtin(Shared::clone(&arena1)).unwrap();
589        #[cfg(not(feature = "sync"))]
590        let size1 = arena1.borrow().len();
591        #[cfg(feature = "sync")]
592        let size1 = arena1.read().unwrap().len();
593
594        let arena2 = Shared::new(SharedCell::new(crate::arena::Arena::new(2048)));
595        token_alloc(
596            &arena2,
597            &Shared::new(Token {
598                kind: TokenKind::Eof,
599                range: Range::default(),
600                module_id: Module::TOP_LEVEL_MODULE_ID,
601            }),
602        );
603        let mut loader2 = ModuleLoader::new(DefaultModuleResolver::default());
604        loader2.load_builtin(Shared::clone(&arena2)).unwrap();
605        #[cfg(not(feature = "sync"))]
606        let size2 = arena2.borrow().len();
607        #[cfg(feature = "sync")]
608        let size2 = arena2.read().unwrap().len();
609
610        assert_eq!(size1, size2, "arena size must match between cache and fresh parse");
611        assert!(size1 > 1, "builtin tokens must be added to the arena");
612    }
613
614    /// The module returned from cache must have the same function/var/macro counts as a fresh parse.
615    #[rstest]
616    fn test_load_builtin_cache_module_counts_consistent(
617        pristine_token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
618    ) {
619        let mut loader1 = ModuleLoader::new(DefaultModuleResolver::default());
620        let module1 = loader1.load_builtin(pristine_token_arena).unwrap();
621
622        let arena2 = Shared::new(SharedCell::new(crate::arena::Arena::new(2048)));
623        token_alloc(
624            &arena2,
625            &Shared::new(Token {
626                kind: TokenKind::Eof,
627                range: Range::default(),
628                module_id: Module::TOP_LEVEL_MODULE_ID,
629            }),
630        );
631        let mut loader2 = ModuleLoader::new(DefaultModuleResolver::default());
632        let module2 = loader2.load_builtin(arena2).unwrap();
633
634        assert_eq!(module1.name, module2.name);
635        assert_eq!(module1.functions.len(), module2.functions.len());
636        assert_eq!(module1.vars.len(), module2.vars.len());
637        assert_eq!(module1.macros.len(), module2.macros.len());
638        assert_eq!(module1.modules.len(), module2.modules.len());
639    }
640
641    /// After load_builtin, the builtin module must be registered at loaded_modules index 1
642    /// (TOP_LEVEL_MODULE is always 0).
643    #[rstest]
644    fn test_load_builtin_module_registered_at_id_one(
645        pristine_token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
646    ) {
647        let mut loader = ModuleLoader::new(DefaultModuleResolver::default());
648        loader.load_builtin(pristine_token_arena).unwrap();
649
650        assert_eq!(loader.loaded_modules.len(), 2);
651        assert!(loader.loaded_modules.contains(Module::BUILTIN_MODULE.into()));
652    }
653
654    /// All tokens injected from cache must carry module_id == 1 (BUILTIN_MODULE_ID),
655    /// so that error diagnostics resolve to the builtin source file rather than garbage.
656    #[rstest]
657    fn test_load_builtin_cache_tokens_have_builtin_module_id(
658        pristine_token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
659    ) {
660        let mut loader1 = ModuleLoader::new(DefaultModuleResolver::default());
661        loader1.load_builtin(pristine_token_arena).unwrap();
662
663        // Second pristine load — this is the cache-hit path.
664        let arena2 = Shared::new(SharedCell::new(crate::arena::Arena::new(2048)));
665        token_alloc(
666            &arena2,
667            &Shared::new(Token {
668                kind: TokenKind::Eof,
669                range: Range::default(),
670                module_id: Module::TOP_LEVEL_MODULE_ID,
671            }),
672        );
673        let mut loader2 = ModuleLoader::new(DefaultModuleResolver::default());
674        loader2.load_builtin(Shared::clone(&arena2)).unwrap();
675
676        let builtin_module_id: crate::ModuleId = 1.into();
677        #[cfg(not(feature = "sync"))]
678        let arena = arena2.borrow();
679        #[cfg(feature = "sync")]
680        let arena = arena2.read().unwrap();
681        for token in arena.as_slice()[1..].iter() {
682            assert_eq!(
683                token.module_id, builtin_module_id,
684                "cached builtin token must have BUILTIN_MODULE_ID"
685            );
686        }
687    }
688
689    #[cfg(feature = "http-import")]
690    #[rstest]
691    #[case("https://example.com/foo.mq")]
692    #[case("http://example.com/foo.mq")]
693    #[case("github.com/alice/mymod")]
694    fn test_resolve_http_blocked_inside_module(#[case] url: &str) {
695        let mut loader = ModuleLoader::new(DefaultModuleResolver::new(vec![]));
696        loader.push_http_boundary();
697        assert!(matches!(loader.resolve(url), Err(ModuleError::HttpImportNotAllowed(_))));
698    }
699
700    #[cfg(feature = "http-import")]
701    #[rstest]
702    #[case("csv")]
703    #[case("local_module")]
704    fn test_resolve_non_http_not_blocked_inside_module(#[case] name: &str) {
705        let mut loader = ModuleLoader::new(DefaultModuleResolver::new(vec![]));
706        loader.push_http_boundary();
707        assert!(!matches!(
708            loader.resolve(name),
709            Err(ModuleError::HttpImportNotAllowed(_))
710        ));
711    }
712
713    #[cfg(feature = "http-import")]
714    #[rstest]
715    #[case("https://example.com/foo.mq")]
716    #[case("github.com/alice/mod")]
717    fn test_pop_http_boundary_restores_access(#[case] url: &str) {
718        let mut loader = ModuleLoader::new(DefaultModuleResolver::new(vec![]));
719        loader.push_http_boundary();
720        assert!(matches!(loader.resolve(url), Err(ModuleError::HttpImportNotAllowed(_))));
721        loader.pop_http_boundary();
722        assert!(!matches!(
723            loader.resolve(url),
724            Err(ModuleError::HttpImportNotAllowed(_))
725        ));
726    }
727
728    #[cfg(feature = "http-import")]
729    #[rstest]
730    #[case("github.com/alice/mod", 2)]
731    #[case("https://example.com/foo.mq", 3)]
732    fn test_nested_boundaries_block_until_all_popped(#[case] url: &str, #[case] depth: usize) {
733        let mut loader = ModuleLoader::new(DefaultModuleResolver::new(vec![]));
734        for _ in 0..depth {
735            loader.push_http_boundary();
736        }
737        assert!(matches!(loader.resolve(url), Err(ModuleError::HttpImportNotAllowed(_))));
738        for _ in 0..depth - 1 {
739            loader.pop_http_boundary();
740            assert!(matches!(loader.resolve(url), Err(ModuleError::HttpImportNotAllowed(_))));
741        }
742        loader.pop_http_boundary();
743        assert!(!matches!(
744            loader.resolve(url),
745            Err(ModuleError::HttpImportNotAllowed(_))
746        ));
747    }
748
749    #[cfg(feature = "http-import")]
750    #[rstest]
751    #[case("https://example.com/foo.mq")]
752    fn test_pop_at_zero_does_not_underflow(#[case] url: &str) {
753        let mut loader = ModuleLoader::new(DefaultModuleResolver::new(vec![]));
754        loader.pop_http_boundary();
755        assert!(!matches!(
756            loader.resolve(url),
757            Err(ModuleError::HttpImportNotAllowed(_))
758        ));
759    }
760}
mq_lang/module.rs

mq_lang/
module.rs
