oak_testing/

lexing.rs

//! Lexer testing utilities for the Oak ecosystem.
//!
//! This module provides comprehensive testing infrastructure for lexers,
//! including file-based testing, expected output comparison, and
//! test result serialization.

use crate::{create_file, source_from_path};
use oak_core::{
    Language, Lexer, Source, TokenType,
    errors::{OakDiagnostics, OakError},
};

#[cfg(feature = "serde")]
use crate::json_from_path;
#[cfg(feature = "serde")]
use serde::Serialize;

use std::{
    path::{Path, PathBuf},
    sync::{Arc, Mutex},
    thread,
    time::{Duration, Instant},
};
use walkdir::WalkDir;

/// A lexer testing utility that can run tests against multiple files.
///
/// The `LexerTester` provides functionality to test lexers against a directory
/// of files with specific extensions, comparing actual output against expected
/// results stored in JSON files.
pub struct LexerTester {
    root: PathBuf,
    extensions: Vec<String>,
    timeout: Duration,
}

/// Expected lexer test results for comparison.
///
/// This struct represents the expected output of a lexer test, including
/// success status, token count, token data, and any expected errors.
#[derive(Debug, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct LexerTestExpected {
    /// Whether the lexing was expected to succeed.
    pub success: bool,
    /// The expected number of tokens.
    pub count: usize,
    /// The expected token data.
    pub tokens: Vec<TokenData>,
    /// Any expected error messages.
    pub errors: Vec<String>,
}

/// Individual token data for lexer testing.
///
/// Represents a single token with its kind, text content, and position
/// information used for testing lexer output.
#[derive(Debug, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct TokenData {
    /// The kind of the token as a string.
    pub kind: String,
    /// The text content of the token.
    pub text: String,
    /// The start position of the token in the source.
    pub start: usize,
    /// The end position of the token in the source.
    pub end: usize,
}

impl LexerTester {
    /// Creates a new lexer tester with the specified root directory.
    pub fn new<P: AsRef<Path>>(root: P) -> Self {
        Self { root: root.as_ref().to_path_buf(), extensions: vec![], timeout: Duration::from_secs(10) }
    }

    /// Adds a file extension to test against.
    pub fn with_extension(mut self, extension: impl ToString) -> Self {
        self.extensions.push(extension.to_string());
        self
    }

    /// Sets the timeout duration for each test.
    pub fn with_timeout(mut self, time: Duration) -> Self {
        self.timeout = time;
        self
    }

    /// Run tests for the given lexer against all files in the root directory with the specified extensions.
    #[cfg(feature = "serde")]
    pub fn run_tests<L, Lex>(self, lexer: &Lex) -> Result<(), OakError>
    where
        L: Language + Send + Sync,
        L::TokenType: serde::Serialize + std::fmt::Debug + Send + Sync,
        Lex: Lexer<L> + Send + Sync + Clone,
    {
        let test_files = self.find_test_files()?;
        let force_regenerated = std::env::var("REGENERATE_TESTS").unwrap_or("0".to_string()) == "1";
        let mut regenerated_any = false;

        for file_path in test_files {
            println!("Testing file: {}", file_path.display());
            regenerated_any |= self.test_single_file::<L, Lex>(&file_path, lexer, force_regenerated)?
        }

        if regenerated_any && force_regenerated {
            println!("Tests regenerated for: {}", self.root.display());
        }

        Ok(())
    }

    /// Run tests for the given lexer against all files in the root directory with the specified extensions.
    #[cfg(not(feature = "serde"))]
    pub fn run_tests<L, Lex>(self, _lexer: &Lex) -> Result<(), OakError>
    where
        L: Language + Send + Sync,
        Lex: Lexer<L> + Send + Sync + Clone,
    {
        Ok(())
    }

    fn find_test_files(&self) -> Result<Vec<PathBuf>, OakError> {
        let mut files = Vec::new();

        for entry in WalkDir::new(&self.root) {
            let entry = entry.unwrap();
            let path = entry.path();

            if path.is_file() {
                if let Some(ext) = path.extension() {
                    let ext_str = ext.to_str().unwrap_or("");
                    if self.extensions.iter().any(|e| e == ext_str) {
                        // Ignore output files generated by the Tester itself to prevent recursive inclusion
                        let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
                        let is_output_file = file_name.ends_with(".parsed.json") || file_name.ends_with(".lexed.json") || file_name.ends_with(".built.json") || file_name.ends_with(".expected.json");

                        if !is_output_file {
                            files.push(path.to_path_buf())
                        }
                    }
                }
            }
        }

        Ok(files)
    }

    #[cfg(feature = "serde")]
    fn test_single_file<L, Lex>(&self, file_path: &Path, lexer: &Lex, force_regenerated: bool) -> Result<bool, OakError>
    where
        L: Language + Send + Sync,
        L::TokenType: serde::Serialize + std::fmt::Debug + Send + Sync,
        Lex: Lexer<L> + Send + Sync + Clone,
    {
        let source = source_from_path(file_path)?;

        // Use Arc and Mutex to share results between threads
        let result = Arc::new(Mutex::new(None));
        let result_clone = Arc::clone(&result);

        // Clone lexer for use in thread
        let lexer_clone = lexer.clone();
        // Wrap source in Arc for sharing between threads
        let source_arc = Arc::new(source);
        let source_clone = Arc::clone(&source_arc);

        // Create a new thread to raise lexical analysis
        std::thread::scope(|s| {
            let handle = s.spawn(move || {
                let mut cache = oak_core::parser::ParseSession::<L>::default();
                let output = lexer_clone.lex(&*source_clone, &[], &mut cache);
                let mut result = result_clone.lock().unwrap();
                *result = Some(output)
            });

            // Wait for thread completion or timeout
            let start_time = Instant::now();
            let timeout_occurred = loop {
                // Check if thread has finished
                if handle.is_finished() {
                    break false;
                }

                // Check for timeout
                if start_time.elapsed() > self.timeout {
                    break true;
                }

                // Sleep briefly to avoid busy waiting
                thread::sleep(Duration::from_millis(10));
            };

            // Return error if timed out
            if timeout_occurred {
                return Err(OakError::custom_error(&format!("Lexer test timed out after {:?} for file: {}", self.timeout, file_path.display())));
            }

            Ok(())
        })?;

        // Get lexical analysis result
        let OakDiagnostics { result: tokens_result, mut diagnostics } = {
            let result_guard = result.lock().unwrap();
            match result_guard.as_ref() {
                Some(output) => output.clone(),
                None => return Err(OakError::custom_error("Failed to get lexer result")),
            }
        };

        // Construct test result
        let mut success = true;
        let tokens = match tokens_result {
            Ok(tokens) => tokens,
            Err(e) => {
                success = false;
                diagnostics.push(e);
                oak_core::Tokens::default()
            }
        };

        if !diagnostics.is_empty() {
            success = false;
        }

        let tokens: Vec<TokenData> = tokens
            .iter()
            .filter(|token| !token.kind.is_ignored())
            .map(|token| {
                let len = source_arc.as_ref().length();
                let start = token.span.start.min(len);
                let end = token.span.end.min(len).max(start);
                let text = source_arc.as_ref().get_text_in((start..end).into()).to_string();
                TokenData { kind: format!("{:?}", token.kind), text, start: token.span.start, end: token.span.end }
            })
            .collect();

        let errors: Vec<String> = diagnostics.iter().map(|e| e.to_string()).collect();
        let test_result = LexerTestExpected { success, count: tokens.len(), tokens, errors };

        // Process expected result file
        let expected_file = file_path.with_extension(format!("{}.lexed.json", file_path.extension().unwrap_or_default().to_str().unwrap_or("")));

        // Migration: If the new naming convention file doesn't exist, but the old one does, rename it
        if !expected_file.exists() {
            let legacy_file = file_path.with_extension("expected.json");
            if legacy_file.exists() {
                let _ = std::fs::rename(&legacy_file, &expected_file);
            }
        }

        let mut regenerated = false;
        if expected_file.exists() && !force_regenerated {
            let expected_json = json_from_path(&expected_file)?;
            let expected: LexerTestExpected = serde_json::from_value(expected_json).map_err(|e| OakError::custom_error(e.to_string()))?;
            if test_result != expected {
                return Err(OakError::test_failure(file_path.to_path_buf(), format!("{:#?}", expected), format!("{:#?}", test_result)));
            }
        }
        else {
            use std::io::Write;
            let mut file = create_file(&expected_file)?;
            let mut buf = Vec::new();
            let formatter = serde_json::ser::PrettyFormatter::with_indent(b"    "); // 4 spaces indentation
            let mut ser = serde_json::Serializer::with_formatter(&mut buf, formatter);
            test_result.serialize(&mut ser).map_err(|e| OakError::custom_error(e.to_string()))?;
            file.write_all(&buf).map_err(|e| OakError::custom_error(e.to_string()))?;
            regenerated = true;
        }

        Ok(regenerated)
    }
}