tytanic_core/test/

mod.rs

//! Test loading and on-disk manipulation.

use std::fmt::Debug;
use std::fs::{self, File};
use std::io::{self, Write};
use std::time::{Duration, Instant};

use ecow::{eco_vec, EcoString, EcoVec};
use thiserror::Error;
use typst::diag::SourceDiagnostic;
use typst::syntax::{FileId, Source, VirtualPath};

use crate::doc;
use crate::doc::{compare, compile, Document, SaveError};
use crate::project::{Paths, Vcs};

mod annotation;
mod id;

pub use self::annotation::{Annotation, ParseAnnotationError};
pub use self::id::{Id, ParseIdError};

// NOTE(tinger): the order of ignoring and deleting/creating documents is not
// random, this is specifically for VCS like jj with active watchman triggers
// and auto snapshotting.
//
// This is currently untested though.

/// The default test input as source code.
pub const DEFAULT_TEST_INPUT: &str = include_str!("default-test.typ");

/// The default test output as an encouded PNG.
pub const DEFAULT_TEST_OUTPUT: &[u8] = include_bytes!("default-test.png");

/// References for a test.
#[derive(Debug, Clone)]
pub enum Reference {
    /// An ephemeral reference script used to compile the reference document on
    /// the fly.
    Ephemeral(EcoString),

    /// Persistent references which are stored on disk.
    Persistent {
        /// The reference document.
        doc: Document,

        /// The optimization options to use when storing the document, `None`
        /// disabled optimization.
        opt: Option<Box<oxipng::Options>>,
    },
}

/// The kind of a unit test.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Kind {
    /// Test is compared to ephemeral references, these are compiled on the fly
    /// from a reference script.
    Ephemeral,

    /// Test is compared to persistent references, these are pre-compiled and
    /// loaded for comparison.
    Persistent,

    /// Test is only compiled.
    CompileOnly,
}

impl Kind {
    /// Whether this kind is is ephemeral.
    pub fn is_ephemeral(self) -> bool {
        matches!(self, Kind::Ephemeral)
    }

    /// Whether this kind is persistent.
    pub fn is_persistent(self) -> bool {
        matches!(self, Kind::Persistent)
    }

    /// Whether this kind is compile-only.
    pub fn is_compile_only(self) -> bool {
        matches!(self, Kind::CompileOnly)
    }

    /// Returns a kebab-case string representing this kind.
    pub fn as_str(self) -> &'static str {
        match self {
            Kind::Ephemeral => "ephemeral",
            Kind::Persistent => "persistent",
            Kind::CompileOnly => "compile-only",
        }
    }
}

impl Reference {
    /// The kind of this reference.
    pub fn kind(&self) -> Kind {
        match self {
            Self::Ephemeral(_) => Kind::Ephemeral,
            Self::Persistent { doc: _, opt: _ } => Kind::Persistent,
        }
    }
}

/// A standalone test script and its assocaited documents.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Test {
    id: Id,
    kind: Kind,
    annotations: EcoVec<Annotation>,
}

impl Test {
    #[cfg(test)]
    pub(crate) fn new_test(id: Id, kind: Kind) -> Self {
        Self {
            id,
            kind,
            annotations: eco_vec![],
        }
    }

    /// Attempt to load a test, returns `None` if no test could be found.
    pub fn load(paths: &Paths, id: Id) -> Result<Option<Test>, LoadError> {
        let test_script = paths.test_script(&id);

        if !test_script.try_exists()? {
            return Ok(None);
        }

        let kind = if paths.test_ref_script(&id).try_exists()? {
            Kind::Ephemeral
        } else if paths.test_ref_dir(&id).try_exists()? {
            Kind::Persistent
        } else {
            Kind::CompileOnly
        };

        let annotations = Annotation::collect(&fs::read_to_string(test_script)?)?;

        Ok(Some(Test {
            id,
            kind,
            annotations,
        }))
    }
}

impl Test {
    /// The id of this test.
    pub fn id(&self) -> &Id {
        &self.id
    }

    /// The kind of this test.
    pub fn kind(&self) -> Kind {
        self.kind
    }

    /// This test's annotations.
    pub fn annotations(&self) -> &[Annotation] {
        &self.annotations
    }

    /// Whether this test has a skip annotation.
    pub fn is_skip(&self) -> bool {
        self.annotations.contains(&Annotation::Skip)
    }
}

impl Test {
    /// Creates a new test on disk, the kind is inferred from the passed
    /// reference and annotations are parsed from the test script.
    pub fn create(
        paths: &Paths,
        vcs: Option<&Vcs>,
        id: Id,
        source: &str,
        reference: Option<Reference>,
    ) -> Result<Test, CreateError> {
        let test_dir = paths.test_dir(&id);
        tytanic_utils::fs::create_dir(test_dir, true)?;

        let mut file = File::options()
            .write(true)
            .create_new(true)
            .open(paths.test_script(&id))?;

        file.write_all(source.as_bytes())?;

        let kind = reference
            .as_ref()
            .map(Reference::kind)
            .unwrap_or(Kind::CompileOnly);

        let annotations = Annotation::collect(source)?;

        let this = Self {
            id,
            kind,
            annotations,
        };

        // ingore temporaries before creating any
        if let Some(vcs) = vcs {
            vcs.ignore(paths, &this)?;
        }

        match reference {
            Some(Reference::Ephemeral(reference)) => {
                this.create_reference_script(paths, reference.as_str())?;
            }
            Some(Reference::Persistent {
                doc: reference,
                opt: options,
            }) => {
                this.create_reference_document(paths, &reference, options.as_deref())?;
            }
            None => {}
        }

        Ok(this)
    }

    /// Creates the temporary directories of this test.
    pub fn create_temporary_directories(&self, paths: &Paths) -> io::Result<()> {
        if self.kind.is_ephemeral() {
            tytanic_utils::fs::remove_dir(paths.test_ref_dir(&self.id), true)?;
            tytanic_utils::fs::create_dir(paths.test_ref_dir(&self.id), true)?;
        }

        tytanic_utils::fs::create_dir(paths.test_out_dir(&self.id), true)?;
        tytanic_utils::fs::create_dir(paths.test_diff_dir(&self.id), true)?;

        Ok(())
    }

    /// Creates the test script of this test, this will truncate the file if it
    /// already exists.
    pub fn create_script(&self, paths: &Paths, source: &str) -> io::Result<()> {
        std::fs::write(paths.test_script(&self.id), source)?;
        Ok(())
    }

    /// Creates reference script of this test, this will truncate the file if it
    /// already exists.
    pub fn create_reference_script(&self, paths: &Paths, source: &str) -> io::Result<()> {
        std::fs::write(paths.test_ref_script(&self.id), source)?;
        Ok(())
    }

    /// Creates the persistent reference document of this test.
    pub fn create_reference_document(
        &self,
        paths: &Paths,
        reference: &Document,
        optimize_options: Option<&oxipng::Options>,
    ) -> Result<(), SaveError> {
        // NOTE(tinger): if there are already more pages than we want to create,
        // the surplus pages would persist and make every comparison fail due to
        // a page count mismatch, so we clear them to be sure.
        self.delete_reference_document(paths)?;

        let ref_dir = paths.test_ref_dir(&self.id);
        tytanic_utils::fs::create_dir(&ref_dir, true)?;
        reference.save(&ref_dir, optimize_options)?;

        Ok(())
    }

    /// Deletes all directories and scripts of this test.
    pub fn delete(&self, paths: &Paths) -> io::Result<()> {
        self.delete_reference_document(paths)?;
        self.delete_reference_script(paths)?;
        self.delete_temporary_directories(paths)?;

        tytanic_utils::fs::remove_file(paths.test_script(&self.id))?;
        tytanic_utils::fs::remove_dir(paths.test_dir(&self.id), true)?;

        Ok(())
    }

    /// Deletes the temporary directories of this test.
    pub fn delete_temporary_directories(&self, paths: &Paths) -> io::Result<()> {
        if !self.kind.is_compile_only() {
            tytanic_utils::fs::remove_dir(paths.test_ref_dir(&self.id), true)?;
        }

        tytanic_utils::fs::remove_dir(paths.test_out_dir(&self.id), true)?;
        tytanic_utils::fs::remove_dir(paths.test_diff_dir(&self.id), true)?;
        Ok(())
    }

    /// Deletes the test script of of this test.
    pub fn delete_script(&self, paths: &Paths) -> io::Result<()> {
        tytanic_utils::fs::remove_file(paths.test_script(&self.id))?;
        Ok(())
    }

    /// Deletes reference script of of this test.
    pub fn delete_reference_script(&self, paths: &Paths) -> io::Result<()> {
        tytanic_utils::fs::remove_file(paths.test_ref_script(&self.id))?;
        Ok(())
    }

    /// Deletes persistent reference document of this test.
    pub fn delete_reference_document(&self, paths: &Paths) -> io::Result<()> {
        tytanic_utils::fs::remove_dir(paths.test_ref_dir(&self.id), true)?;
        Ok(())
    }

    /// Removes any previous references, if they exist and creates a reference
    /// script by copying the test script.
    pub fn make_ephemeral(&mut self, paths: &Paths, vcs: Option<&Vcs>) -> io::Result<()> {
        self.kind = Kind::Ephemeral;

        // ensure deletion is recorded before ignore file is updated
        self.delete_reference_script(paths)?;
        self.delete_reference_document(paths)?;

        if let Some(vcs) = vcs {
            vcs.ignore(paths, self)?;
        }

        // copy refernces after ignore file is updated
        std::fs::copy(paths.test_script(&self.id), paths.test_ref_script(&self.id))?;

        Ok(())
    }

    /// Removes any previous references, if they exist and creates persistent
    /// references from the given pages.
    pub fn make_persistent(
        &mut self,
        paths: &Paths,
        vcs: Option<&Vcs>,
        reference: &Document,
        optimize_options: Option<&oxipng::Options>,
    ) -> Result<(), SaveError> {
        self.kind = Kind::Persistent;

        // ensure deletion/creation is recorded before ignore file is updated
        self.delete_reference_script(paths)?;
        self.create_reference_document(paths, reference, optimize_options)?;

        if let Some(vcs) = vcs {
            vcs.ignore(paths, self)?;
        }

        Ok(())
    }

    /// Removes any previous references, if they exist.
    pub fn make_compile_only(&mut self, paths: &Paths, vcs: Option<&Vcs>) -> io::Result<()> {
        self.kind = Kind::CompileOnly;

        // ensure deletion is recorded before ignore file is updated
        self.delete_reference_document(paths)?;
        self.delete_reference_script(paths)?;

        if let Some(vcs) = vcs {
            vcs.ignore(paths, self)?;
        }

        Ok(())
    }

    /// Loads the test script source of this test.
    pub fn load_source(&self, paths: &Paths) -> io::Result<Source> {
        let test_script = paths.test_script(&self.id);

        Ok(Source::new(
            FileId::new(
                None,
                VirtualPath::new(
                    test_script
                        .strip_prefix(paths.project_root())
                        .unwrap_or(&test_script),
                ),
            ),
            std::fs::read_to_string(test_script)?,
        ))
    }

    /// Loads the reference test script source of this test, if this test is
    /// ephemeral.
    pub fn load_reference_source(&self, paths: &Paths) -> io::Result<Option<Source>> {
        if !self.kind().is_ephemeral() {
            return Ok(None);
        }

        let ref_script = paths.test_ref_script(&self.id);
        Ok(Some(Source::new(
            FileId::new(
                None,
                VirtualPath::new(
                    ref_script
                        .strip_prefix(paths.project_root())
                        .unwrap_or(&ref_script),
                ),
            ),
            std::fs::read_to_string(ref_script)?,
        )))
    }

    /// Loads the test document of this test.
    pub fn load_document(&self, paths: &Paths) -> Result<Document, doc::LoadError> {
        Document::load(paths.test_out_dir(&self.id))
    }

    /// Loads the persistent reference document of this test.
    pub fn load_reference_document(&self, paths: &Paths) -> Result<Document, doc::LoadError> {
        Document::load(paths.test_ref_dir(&self.id))
    }
}

/// Returned by [`Test::create`].
#[derive(Debug, Error)]
pub enum CreateError {
    /// An error occurred while parsing a test annotation.
    #[error("an error occurred while parsing a test annotation")]
    Annotation(#[from] ParseAnnotationError),

    /// An error occurred while saving test files.
    #[error("an error occurred while saving test files")]
    Save(#[from] doc::SaveError),

    /// An io error occurred.
    #[error("an io error occurred")]
    Io(#[from] io::Error),
}

/// Returned by [`Test::load`].
#[derive(Debug, Error)]
pub enum LoadError {
    /// An error occurred while parsing a test annotation.
    #[error("an error occurred while parsing a test annotation")]
    Annotation(#[from] ParseAnnotationError),

    /// An io error occurred.
    #[error("an io error occurred")]
    Io(#[from] io::Error),
}

/// The stage of a single test run.
#[derive(Debug, Clone, Default)]
pub enum Stage {
    /// The test was cancelled or not started in the first place.
    #[default]
    Skipped,

    /// The test was filtered out by a [`Filter`].
    ///
    /// [`Filter`]: crate::suite::Filter
    Filtered,

    /// The test failed compilation.
    FailedCompilation {
        /// The inner error.
        error: compile::Error,

        /// Whether this was a compilation failure of the reference.
        reference: bool,
    },

    /// The test passed compilation, but failed comparison.
    FailedComparison(compare::Error),

    /// The test passed compilation, but did not run comparison.
    PassedCompilation,

    /// The test passed compilation and comparison.
    PassedComparison,
}

/// The result of a single test run.
#[derive(Debug, Clone)]
pub struct TestResult {
    stage: Stage,
    warnings: EcoVec<SourceDiagnostic>,
    timestamp: Instant,
    duration: Duration,
}

impl TestResult {
    /// Create a result for a test for a skipped test. This will set the
    /// starting time to now, the duration to zero and the result to `None`.
    ///
    /// This can be used for constructing test results in advance to ensure an
    /// aborted test run contains a skip result for all yet-to-be-run tests.
    pub fn skipped() -> Self {
        Self {
            stage: Stage::Skipped,
            warnings: eco_vec![],
            timestamp: Instant::now(),
            duration: Duration::ZERO,
        }
    }

    /// Create a result for a test for a filtered test. This will set the
    /// starting time to now, the duration to zero and the result to filtered.
    pub fn filtered() -> Self {
        Self {
            stage: Stage::Filtered,
            warnings: eco_vec![],
            timestamp: Instant::now(),
            duration: Duration::ZERO,
        }
    }
}

impl TestResult {
    /// The stage of this rest result, if it was started.
    pub fn stage(&self) -> &Stage {
        &self.stage
    }

    /// The warnings of the test emitted by the compiler.
    pub fn warnings(&self) -> &[SourceDiagnostic] {
        &self.warnings
    }

    /// The timestamp at which the suite run started.
    pub fn timestamp(&self) -> Instant {
        self.timestamp
    }

    /// The duration of the test, this a zero if this test wasn't started.
    pub fn duration(&self) -> Duration {
        self.duration
    }

    /// Whether the test was not started.
    pub fn is_skipped(&self) -> bool {
        matches!(&self.stage, Stage::Skipped)
    }

    /// Whether the test was filtered out.
    pub fn is_filtered(&self) -> bool {
        matches!(&self.stage, Stage::Filtered)
    }

    /// Whether the test passed compilation or comparison.
    pub fn is_pass(&self) -> bool {
        matches!(
            &self.stage,
            Stage::PassedCompilation | Stage::PassedComparison
        )
    }

    /// Whether the test failed compilation or comparison.
    pub fn is_fail(&self) -> bool {
        matches!(
            &self.stage,
            Stage::FailedCompilation { .. } | Stage::FailedComparison(..),
        )
    }

    /// The errors emitted by the compiler if compilation failed.
    pub fn errors(&self) -> Option<&[SourceDiagnostic]> {
        match &self.stage {
            Stage::FailedCompilation { error, .. } => Some(&error.0),
            _ => None,
        }
    }
}

impl TestResult {
    /// Sets the timestamp to [`Instant::now`].
    ///
    /// See [`TestResult::end`].
    pub fn start(&mut self) {
        self.timestamp = Instant::now();
    }

    /// Sets the duration to the time elapsed since [`TestResult::start`] was
    /// called.
    pub fn end(&mut self) {
        self.duration = self.timestamp.elapsed();
    }

    /// Sets the kind for this test to a compilation pass.
    pub fn set_passed_compilation(&mut self) {
        self.stage = Stage::PassedCompilation;
    }

    /// Sets the kind for this test to a reference compilation failure.
    pub fn set_failed_reference_compilation(&mut self, error: compile::Error) {
        self.stage = Stage::FailedCompilation {
            error,
            reference: true,
        };
    }

    /// Sets the kind for this test to a test compilation failure.
    pub fn set_failed_test_compilation(&mut self, error: compile::Error) {
        self.stage = Stage::FailedCompilation {
            error,
            reference: false,
        };
    }

    /// Sets the kind for this test to a test comparison pass.
    pub fn set_passed_comparison(&mut self) {
        self.stage = Stage::PassedComparison;
    }

    /// Sets the kind for this test to a comparison failure.
    pub fn set_failed_comparison(&mut self, error: compare::Error) {
        self.stage = Stage::FailedComparison(error);
    }

    /// Sets the warnings for this test.
    pub fn set_warnings<I>(&mut self, warnings: I)
    where
        I: Into<EcoVec<SourceDiagnostic>>,
    {
        self.warnings = warnings.into();
    }
}

impl Default for TestResult {
    fn default() -> Self {
        Self::skipped()
    }
}

#[cfg(test)]
mod tests {
    use tytanic_utils::fs::{Setup, TempTestEnv};

    use super::*;

    fn id(id: &str) -> Id {
        Id::new(id).unwrap()
    }

    fn test(test_id: &str, kind: Kind) -> Test {
        Test::new_test(id(test_id), kind)
    }

    fn setup_all(root: &mut Setup) -> &mut Setup {
        root.setup_file("tests/compile-only/test.typ", "Hello World")
            .setup_file("tests/ephemeral/test.typ", "Hello World")
            .setup_file("tests/ephemeral/ref.typ", "Hello\nWorld")
            .setup_file("tests/persistent/test.typ", "Hello World")
            .setup_dir("tests/persistent/ref")
    }

    #[test]
    fn test_create() {
        TempTestEnv::run(
            |root| root.setup_dir("tests"),
            |root| {
                let paths = Paths::new(root, None);
                Test::create(&paths, None, id("compile-only"), "Hello World", None).unwrap();

                Test::create(
                    &paths,
                    None,
                    id("ephemeral"),
                    "Hello World",
                    Some(Reference::Ephemeral("Hello\nWorld".into())),
                )
                .unwrap();

                Test::create(
                    &paths,
                    None,
                    id("persistent"),
                    "Hello World",
                    Some(Reference::Persistent {
                        doc: Document::new(vec![]),
                        opt: None,
                    }),
                )
                .unwrap();
            },
            |root| {
                root.expect_file_content("tests/compile-only/test.typ", "Hello World")
                    .expect_file_content("tests/ephemeral/test.typ", "Hello World")
                    .expect_file_content("tests/ephemeral/ref.typ", "Hello\nWorld")
                    .expect_file_content("tests/persistent/test.typ", "Hello World")
                    .expect_dir("tests/persistent/ref")
            },
        );
    }

    #[test]
    fn test_make_ephemeral() {
        TempTestEnv::run(
            setup_all,
            |root| {
                let paths = Paths::new(root, None);
                test("compile-only", Kind::CompileOnly)
                    .make_ephemeral(&paths, None)
                    .unwrap();
                test("ephemeral", Kind::Ephemeral)
                    .make_ephemeral(&paths, None)
                    .unwrap();
                test("persistent", Kind::Persistent)
                    .make_ephemeral(&paths, None)
                    .unwrap();
            },
            |root| {
                root.expect_file_content("tests/compile-only/test.typ", "Hello World")
                    .expect_file_content("tests/compile-only/ref.typ", "Hello World")
                    .expect_file_content("tests/ephemeral/test.typ", "Hello World")
                    .expect_file_content("tests/ephemeral/ref.typ", "Hello World")
                    .expect_file_content("tests/persistent/test.typ", "Hello World")
                    .expect_file_content("tests/persistent/ref.typ", "Hello World")
            },
        );
    }

    #[test]
    fn test_make_persistent() {
        TempTestEnv::run(
            setup_all,
            |root| {
                let paths = Paths::new(root, None);
                test("compile-only", Kind::CompileOnly)
                    .make_persistent(&paths, None, &Document::new([]), None)
                    .unwrap();

                test("ephemeral", Kind::Ephemeral)
                    .make_persistent(&paths, None, &Document::new([]), None)
                    .unwrap();

                test("persistent", Kind::Persistent)
                    .make_persistent(&paths, None, &Document::new([]), None)
                    .unwrap();
            },
            |root| {
                root.expect_file_content("tests/compile-only/test.typ", "Hello World")
                    .expect_dir("tests/compile-only/ref")
                    .expect_file_content("tests/ephemeral/test.typ", "Hello World")
                    .expect_dir("tests/ephemeral/ref")
                    .expect_file_content("tests/persistent/test.typ", "Hello World")
                    .expect_dir("tests/persistent/ref")
            },
        );
    }

    #[test]
    fn test_make_compile_only() {
        TempTestEnv::run(
            setup_all,
            |root| {
                let paths = Paths::new(root, None);
                test("compile-only", Kind::CompileOnly)
                    .make_compile_only(&paths, None)
                    .unwrap();

                test("ephemeral", Kind::Ephemeral)
                    .make_compile_only(&paths, None)
                    .unwrap();

                test("persistent", Kind::Persistent)
                    .make_compile_only(&paths, None)
                    .unwrap();
            },
            |root| {
                root.expect_file_content("tests/compile-only/test.typ", "Hello World")
                    .expect_file_content("tests/ephemeral/test.typ", "Hello World")
                    .expect_file_content("tests/persistent/test.typ", "Hello World")
            },
        );
    }

    #[test]
    fn test_load_sources() {
        TempTestEnv::run_no_check(
            |root| {
                root.setup_file("tests/fancy/test.typ", "Hello World")
                    .setup_file("tests/fancy/ref.typ", "Hello\nWorld")
            },
            |root| {
                let paths = Paths::new(root, None);

                let mut test = test("fancy", Kind::Ephemeral);
                test.kind = Kind::Ephemeral;

                test.load_source(&paths).unwrap();
                test.load_reference_source(&paths).unwrap().unwrap();
            },
        );
    }

    #[test]
    fn test_sources_virtual() {
        TempTestEnv::run_no_check(
            |root| root.setup_file_empty("tests/fancy/test.typ"),
            |root| {
                let paths = Paths::new(root, None);

                let test = test("fancy", Kind::CompileOnly);

                let source = test.load_source(&paths).unwrap();
                assert_eq!(
                    source.id().vpath().resolve(root).unwrap(),
                    root.join("tests/fancy/test.typ")
                );
            },
        );
    }
}