judge_framework/backend/

mod.rs

mod error;
mod runner;
mod sandbox;
mod util;

use std::{ffi::CStr, path::PathBuf, time::Duration};

use const_format::formatcp;
pub use error::{Error, Result};
use nix::sys::resource::Resource;
pub use runner::Runner;
pub use sandbox::RlimitConfig;

pub struct TestCase {
    input_path: PathBuf,
    output_path: PathBuf,
}

#[derive(Debug, PartialEq)]
pub enum JudgeVerdict {
    Accepted,
    WrongAnswer { diff: String },
    TimeLimitExceeded,
    RuntimeError,
}

#[derive(Debug)]
pub struct JudgeResult {
    pub verdict: JudgeVerdict,
    pub run_time: Duration,
}

#[derive(Debug, Clone, Copy)]
pub struct Command {
    pub binary: &'static str,
    pub args: &'static [&'static str],
}

#[derive(Debug, Clone, Copy)]
pub struct RawCommand {
    pub binary: &'static CStr,
    pub args: &'static [&'static CStr],
}

const MAIN_FILE_NAME: &str = "Main";

const DEFAULT_RLIMIT_CONFIGS: [RlimitConfig; 5] = [
    RlimitConfig {
        resource: Resource::RLIMIT_STACK,
        soft_limit: 1024 * 1024 * 1024 * 1024,
        hard_limit: 1024 * 1024 * 1024 * 1024,
    },
    RlimitConfig {
        resource: Resource::RLIMIT_AS,
        soft_limit: 1024 * 1024 * 1024 * 1024,
        hard_limit: 1024 * 1024 * 1024 * 1024,
    },
    RlimitConfig {
        resource: Resource::RLIMIT_CPU,
        soft_limit: 60,
        hard_limit: 90,
    },
    RlimitConfig {
        resource: Resource::RLIMIT_NPROC,
        soft_limit: 1,
        hard_limit: 1,
    },
    RlimitConfig {
        resource: Resource::RLIMIT_FSIZE,
        soft_limit: 1024,
        hard_limit: 1024,
    },
];

const DEFAULT_SCMP_BLACK_LIST: [&str; 4] = ["clone", "clone3", "fork", "vfork"];

pub const CPP_RUNNER: Runner = Runner {
    extension: "cpp",
    compile_command: Some(Command {
        binary: "g++",
        args: &["-o", MAIN_FILE_NAME, formatcp!("{MAIN_FILE_NAME}.cpp")],
    }),
    run_command: RawCommand {
        //WARN: hard coded binary name
        binary: c"./Main",
        args: &[],
    },
    time_out: 2,
    rlimit_config: &DEFAULT_RLIMIT_CONFIGS,
    scmp_black_list: &DEFAULT_SCMP_BLACK_LIST,

    #[cfg(test)]
    infinite_loop_code: r#"
int main() {
    while(1) {}
}
    "#,
    #[cfg(test)]
    runtime_error_code: r#"
#include <bits/stdc++.h>

using namespace std;

int main() {
    vector<int> a;
    a[1] = 0;
}
    "#,
    #[cfg(test)]
    compilation_error_code: "0eae2a2a-8f74-43f6-b486-646ce38f6d21",
};

pub const JAVA_RUNNER: Runner = Runner {
    extension: "java",
    compile_command: Some(Command {
        binary: "javac",
        args: &[formatcp!("{MAIN_FILE_NAME}.java")],
    }),
    run_command: RawCommand {
        binary: c"java",
        //WARN: hard coded entry file name
        args: &[c"java", c"Main"],
    },
    time_out: 4,
    // remove nproc limit since JVM will need to create thread to start
    rlimit_config: &[
        RlimitConfig {
            resource: Resource::RLIMIT_STACK,
            soft_limit: 1024 * 1024 * 1024 * 1024,
            hard_limit: 1024 * 1024 * 1024 * 1024,
        },
        RlimitConfig {
            resource: Resource::RLIMIT_AS,
            soft_limit: 1024 * 1024 * 1024 * 1024,
            hard_limit: 1024 * 1024 * 1024 * 1024,
        },
        RlimitConfig {
            resource: Resource::RLIMIT_CPU,
            soft_limit: 60,
            hard_limit: 90,
        },
        RlimitConfig {
            resource: Resource::RLIMIT_FSIZE,
            soft_limit: 1024,
            hard_limit: 1024,
        },
    ],
    scmp_black_list: &["fork", "vfork"],

    #[cfg(test)]
    infinite_loop_code: r#"
class Main {
    public static void main(String[] args) {
        while(true) {}
    }
}
    "#,
    #[cfg(test)]
    runtime_error_code: r#"
class Main {
    public static void main(String[] args) {
        throw new RuntimeException("Test runtime exception");
    }
}
    "#,
    #[cfg(test)]
    compilation_error_code: "0eae2a2a-8f74-43f6-b486-646ce38f6d21",
};

pub const PYTHON_RUNNER: Runner = Runner {
    extension: "py",
    compile_command: None,
    run_command: RawCommand {
        binary: c"python",
        //WARN: hard coded entry file name
        args: &[c"python", c"Main.py"],
    },
    time_out: 10,
    rlimit_config: &DEFAULT_RLIMIT_CONFIGS,
    scmp_black_list: &DEFAULT_SCMP_BLACK_LIST,

    #[cfg(test)]
    infinite_loop_code: r#"
i = 0
while True:
    i = 1
    "#,
    #[cfg(test)]
    runtime_error_code: "3ff926be-8e1c-4637-a248-58405ccf04e0",
    #[cfg(test)]
    compilation_error_code: "",
};

#[cfg(test)]
mod tests {
    use std::assert_matches::assert_matches;
    use std::fs;
    use std::path::{Path, PathBuf};

    use rstest::rstest;

    use super::*;

    impl TestCase {
        pub fn null() -> Self {
            TestCase {
                input_path: Path::new("/dev/null").to_path_buf(),
                output_path: Path::new("/dev/null").to_path_buf(),
            }
        }
    }

    fn read_code(problem_path: PathBuf, extension: &str) -> String {
        let mut code_path = problem_path.clone();
        code_path.push("code");
        code_path.set_extension(extension);

        String::from_utf8(fs::read(code_path).unwrap()).unwrap()
    }

    fn read_test_cases(problem_path: PathBuf) -> Vec<TestCase> {
        let test_cases_dir = problem_path.join("test_cases");
        fs::read_dir(test_cases_dir)
            .unwrap()
            .flatten()
            .map(|test_case_dir| test_case_dir.path())
            .map(|test_case_dir| TestCase {
                input_path: test_case_dir.join("in.txt"),
                output_path: test_case_dir.join("out.txt"),
            })
            .collect()
    }

    #[rstest]
    #[trace]
    #[tokio::test]
    async fn test_should_pass_all(
        #[values(CPP_RUNNER, JAVA_RUNNER, PYTHON_RUNNER)] runner: Runner,
        #[files("test/data/backend/**")]
        #[exclude("test_cases")]
        problem_path: PathBuf,
    ) {
        let code = read_code(problem_path.clone(), runner.extension);
        let test_cases = read_test_cases(problem_path);

        let details = runner.run(code.as_str(), test_cases).await.unwrap();

        let is_all_passed = details
            .iter()
            .all(|detail| detail.verdict == JudgeVerdict::Accepted);
        assert!(is_all_passed)
    }

    #[rstest]
    #[trace]
    #[tokio::test]
    async fn test_should_fail_all(
        #[values(CPP_RUNNER, JAVA_RUNNER, PYTHON_RUNNER)] runner: Runner,
        #[files("test/data/backend/**")]
        #[exclude("test_cases")]
        problem_path: PathBuf,
    ) {
        let code = read_code(problem_path.clone(), runner.extension);
        let test_cases = read_test_cases(problem_path)
            .into_iter()
            .map(|mut test_case| {
                test_case.output_path = Path::new("/dev/null").to_path_buf();
                test_case
            })
            .collect();

        let details = runner.run(code.as_str(), test_cases).await.unwrap();

        for detail in details {
            assert_matches!(detail.verdict, JudgeVerdict::WrongAnswer { diff: _ });
        }
    }

    #[rstest]
    #[trace]
    #[tokio::test]
    async fn test_should_timed_out(
        #[values(CPP_RUNNER, JAVA_RUNNER, PYTHON_RUNNER)] runner: Runner,
    ) {
        let code = runner.infinite_loop_code;
        let test_cases = vec![TestCase::null()];

        let details = runner.run(code, test_cases).await.unwrap();

        let is_all_timed_out = details
            .iter()
            .all(|detail| detail.verdict == JudgeVerdict::TimeLimitExceeded);
        assert!(is_all_timed_out)
    }

    #[rstest]
    #[trace]
    #[tokio::test]
    async fn test_should_compile_fail(#[values(CPP_RUNNER, JAVA_RUNNER)] runner: Runner) {
        let code = &runner.compilation_error_code;
        let test_cases = vec![];

        assert_matches!(
            runner.run(code, test_cases).await,
            Err(Error::Compilation { message: _ })
        );
    }

    #[rstest]
    #[trace]
    #[tokio::test]
    async fn test_should_error_at_runtime(
        #[values(CPP_RUNNER, JAVA_RUNNER, PYTHON_RUNNER)] runner: Runner,
    ) {
        let code = runner.runtime_error_code;
        let test_cases = vec![TestCase::null()];

        let details = runner.run(code, test_cases).await.unwrap();

        let is_all_runtime_error = details
            .iter()
            .all(|detail| detail.verdict == JudgeVerdict::RuntimeError);
        assert!(is_all_runtime_error)
    }
}