oak_testing/
lexing.rs

1//! Lexer testing utilities for the Oak ecosystem.
2//!
3//! This module provides comprehensive testing infrastructure for lexers,
4//! including file-based testing, expected output comparison, and
5//! test result serialization.
6
7use crate::{create_file, json_from_path, source_from_path};
8use oak_core::{
9    Language, Lexer, Source, TokenType,
10    errors::{OakDiagnostics, OakError},
11};
12use serde::{Deserialize, Serialize};
13
14use std::{
15    path::{Path, PathBuf},
16    sync::{Arc, Mutex},
17    thread,
18    time::{Duration, Instant},
19};
20use walkdir::WalkDir;
21
22/// A lexer testing utility that can run tests against multiple files.
23///
24/// The `LexerTester` provides functionality to test lexers against a directory
25/// of files with specific extensions, comparing actual output against expected
26/// results stored in JSON files.
27pub struct LexerTester {
28    root: PathBuf,
29    extensions: Vec<String>,
30    timeout: Duration,
31}
32
33/// Expected lexer test results for comparison.
34///
35/// This struct represents the expected output of a lexer test, including
36/// success status, token count, token data, and any expected errors.
37#[derive(Debug, PartialEq, Serialize, Deserialize)]
38pub struct LexerTestExpected {
39    pub success: bool,
40    pub count: usize,
41    pub tokens: Vec<TokenData>,
42    pub errors: Vec<String>,
43}
44
45/// Individual token data for lexer testing.
46///
47/// Represents a single token with its kind, text content, and position
48/// information used for testing lexer output.
49#[derive(Debug, PartialEq, Serialize, Deserialize)]
50pub struct TokenData {
51    pub kind: String,
52    pub text: String,
53    pub start: usize,
54    pub end: usize,
55}
56
57impl LexerTester {
58    /// Creates a new lexer tester with the specified root directory.
59    pub fn new<P: AsRef<Path>>(root: P) -> Self {
60        Self { root: root.as_ref().to_path_buf(), extensions: vec![], timeout: Duration::from_secs(10) }
61    }
62
63    /// Adds a file extension to test against.
64    pub fn with_extension(mut self, extension: impl ToString) -> Self {
65        self.extensions.push(extension.to_string());
66        self
67    }
68    /// Sets the timeout duration for each test.
69    pub fn with_timeout(mut self, time: Duration) -> Self {
70        self.timeout = time;
71        self
72    }
73
74    /// Run tests for the given lexer against all files in the root directory with the specified extensions.
75    pub fn run_tests<L, Lex>(self, lexer: &Lex) -> Result<(), OakError>
76    where
77        L: Language + Send + Sync,
78        L::TokenType: Serialize + std::fmt::Debug + Send + Sync,
79        Lex: Lexer<L> + Send + Sync + Clone,
80    {
81        let test_files = self.find_test_files()?;
82        let force_regenerated = std::env::var("REGENERATE_TESTS").unwrap_or("0".to_string()) == "1";
83        let mut regenerated_any = false;
84
85        for file_path in test_files {
86            println!("Testing file: {}", file_path.display());
87            regenerated_any |= self.test_single_file::<L, Lex>(&file_path, lexer, force_regenerated)?
88        }
89
90        if regenerated_any && force_regenerated {
91            println!("Tests regenerated for: {}", self.root.display());
92            Ok(())
93        }
94        else {
95            Ok(())
96        }
97    }
98
99    fn find_test_files(&self) -> Result<Vec<PathBuf>, OakError> {
100        let mut files = Vec::new();
101
102        for entry in WalkDir::new(&self.root) {
103            let entry = entry.unwrap();
104            let path = entry.path();
105
106            if path.is_file() {
107                if let Some(ext) = path.extension() {
108                    let ext_str = ext.to_str().unwrap_or("");
109                    if self.extensions.iter().any(|e| e == ext_str) {
110                        // Ignore output files generated by the Tester itself to prevent recursive inclusion
111                        let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
112                        let is_output_file = file_name.ends_with(".parsed.json") || file_name.ends_with(".lexed.json") || file_name.ends_with(".built.json") || file_name.ends_with(".expected.json");
113
114                        if !is_output_file {
115                            files.push(path.to_path_buf())
116                        }
117                    }
118                }
119            }
120        }
121
122        Ok(files)
123    }
124
125    fn test_single_file<L, Lex>(&self, file_path: &Path, lexer: &Lex, force_regenerated: bool) -> Result<bool, OakError>
126    where
127        L: Language + Send + Sync,
128        L::TokenType: Serialize + std::fmt::Debug + Send + Sync,
129        Lex: Lexer<L> + Send + Sync + Clone,
130    {
131        let source = source_from_path(file_path)?;
132
133        // Use Arc and Mutex to share results between threads
134        let result = Arc::new(Mutex::new(None));
135        let result_clone = Arc::clone(&result);
136
137        // Clone lexer for use in thread
138        let lexer_clone = lexer.clone();
139        // Wrap source in Arc for sharing between threads
140        let source_arc = Arc::new(source);
141        let source_clone = Arc::clone(&source_arc);
142
143        // Create a new thread to raise lexical analysis
144        std::thread::scope(|s| {
145            let handle = s.spawn(move || {
146                let mut cache = oak_core::parser::ParseSession::<L>::default();
147                let output = lexer_clone.lex(&*source_clone, &[], &mut cache);
148                let mut result = result_clone.lock().unwrap();
149                *result = Some(output)
150            });
151
152            // Wait for thread completion or timeout
153            let start_time = Instant::now();
154            let timeout_occurred = loop {
155                // Check if thread has finished
156                if handle.is_finished() {
157                    break false;
158                }
159
160                // Check for timeout
161                if start_time.elapsed() > self.timeout {
162                    break true;
163                }
164
165                // Sleep briefly to avoid busy waiting
166                thread::sleep(Duration::from_millis(10));
167            };
168
169            // Return error if timed out
170            if timeout_occurred {
171                return Err(OakError::custom_error(&format!("Lexer test timed out after {:?} for file: {}", self.timeout, file_path.display())));
172            }
173
174            Ok(())
175        })?;
176
177        // Get lexical analysis result
178        let OakDiagnostics { result: tokens_result, mut diagnostics } = {
179            let result_guard = result.lock().unwrap();
180            match result_guard.as_ref() {
181                Some(output) => output.clone(),
182                None => return Err(OakError::custom_error("Failed to get lexer result")),
183            }
184        };
185
186        // Construct test result
187        let mut success = true;
188        let tokens = match tokens_result {
189            Ok(tokens) => tokens,
190            Err(e) => {
191                success = false;
192                diagnostics.push(e);
193                triomphe::Arc::from_iter(Vec::new())
194            }
195        };
196
197        if !diagnostics.is_empty() {
198            success = false;
199        }
200
201        let tokens: Vec<TokenData> = tokens
202            .iter()
203            .filter(|token| !token.kind.is_ignored())
204            .map(|token| {
205                let len = source_arc.as_ref().length();
206                let start = token.span.start.min(len);
207                let end = token.span.end.min(len).max(start);
208                let text = source_arc.as_ref().get_text_in((start..end).into()).to_string();
209                TokenData { kind: format!("{:?}", token.kind), text, start: token.span.start, end: token.span.end }
210            })
211            .collect();
212
213        let errors: Vec<String> = diagnostics.iter().map(|e| e.to_string()).collect();
214        let test_result = LexerTestExpected { success, count: tokens.len(), tokens, errors };
215
216        // Process expected result file
217        let expected_file = file_path.with_extension(format!("{}.lexed.json", file_path.extension().unwrap_or_default().to_str().unwrap_or("")));
218
219        // Migration: If the new naming convention file doesn't exist, but the old one does, rename it
220        if !expected_file.exists() {
221            let legacy_file = file_path.with_extension("expected.json");
222            if legacy_file.exists() {
223                let _ = std::fs::rename(&legacy_file, &expected_file);
224            }
225        }
226
227        let mut regenerated = false;
228        if expected_file.exists() && !force_regenerated {
229            let expected_json = json_from_path(&expected_file)?;
230            let expected: LexerTestExpected = serde_json::from_value(expected_json).map_err(|e| OakError::custom_error(e.to_string()))?;
231            if test_result != expected {
232                return Err(OakError::test_failure(file_path.to_path_buf(), format!("{:#?}", expected), format!("{:#?}", test_result)));
233            }
234        }
235        else {
236            use std::io::Write;
237            let mut file = create_file(&expected_file)?;
238            let json_val = serde_json::to_string_pretty(&test_result).map_err(|e| OakError::custom_error(e.to_string()))?;
239            file.write_all(json_val.as_bytes()).map_err(|e| OakError::custom_error(e.to_string()))?;
240            regenerated = true;
241        }
242
243        Ok(regenerated)
244    }
245}
oak_testing/lexing.rs

oak_testing/
lexing.rs
