Trait deqp_runner::TestCommand

source · 
pub trait TestCommand: Send + Sync {

Show 22 methods    // Required methods
    fn name(&self) -> &str;
    fn config(&self) -> &TestConfiguration;
    fn prepare(
        &self,
        caselist_state: &CaselistState,
        tests: &[&TestCase]
    ) -> Result<Command>;
    fn parse_results(
        &self,
        caselist_state: &CaselistState,
        tests: &[&TestCase],
        stdout: TimeoutChildStdout<'_>,
        timer: Option<Timer>
    ) -> Result<CaselistResult>;

    // Provided methods
    fn see_more(&self, _name: &str, _caselist_state: &CaselistState) -> String { ... }
    fn skips(&self) -> &RegexSet { ... }
    fn flakes(&self) -> &RegexSet { ... }
    fn baseline(&self) -> &RunnerResults { ... }
    fn baseline_status(&self, test: &str) -> Option<RunnerStatus> { ... }
    fn translate_result(
        &self,
        result: &TestResult,
        caselist_state: &CaselistState
    ) -> RunnerStatus { ... }
    fn skip_test(&self, test: &str) -> bool { ... }
    fn handle_result(
        &self,
        _caselist_state: &CaselistState,
        _result: &TestResult,
        _status: &RunnerStatus
    ) -> Result<()> { ... }
    fn log_path(
        &self,
        caselist_state: &CaselistState,
        _tests: &[&TestCase]
    ) -> Result<PathBuf> { ... }
    fn should_save_log(
        &self,
        _caselist_state: &CaselistState,
        _tests: &[&TestCase]
    ) -> bool { ... }
    fn clean(
        &self,
        _caselist_state: &CaselistState,
        _tests: &[&TestCase],
        _results: &[RunnerResult]
    ) -> Result<()> { ... }
    fn handle_exit_status(
        &self,
        code: Option<i32>,
        some_result: Option<&mut TestResult>
    ) { ... }
    fn run(
        &self,
        caselist_state: &CaselistState,
        tests: &[&TestCase]
    ) -> Result<Vec<RunnerResult>> { ... }
    fn run_caselist_and_flake_detect(
        &self,
        caselist: &[TestCase],
        caselist_state: &mut CaselistState
    ) -> Result<Vec<RunnerResult>> { ... }
    fn process_caselist(
        &self,
        tests: Vec<TestCase>,
        caselist_id: u32
    ) -> Result<Vec<RunnerResult>> { ... }
    fn split_tests_to_groups(
        &self,
        tests: Vec<TestCase>,
        tests_per_group: usize,
        min_tests_per_group: usize,
        sub_config: &SubRunConfig,
        include_filters: &[RegexSet]
    ) -> Result<Vec<(&dyn TestCommand, Vec<TestCase>)>>
       where Self: Sized { ... }
    fn caselist_file_path(
        &self,
        caselist_state: &CaselistState,
        suffix: &str
    ) -> Result<PathBuf> { ... }
    fn prefix(&self) -> &str { ... }

}
Expand description

This is implemented by each supported test suite.

Required Methods§

source

fn name(&self) -> &str

Must be implemented, returns the name of this TestCommand for logging errors.

source

fn config(&self) -> &TestConfiguration

Must be implemented, returns the shared test configuration for this TestCommand.

source

fn prepare( &self, caselist_state: &CaselistState, tests: &[&TestCase] ) -> Result<Command>

source

fn parse_results( &self, caselist_state: &CaselistState, tests: &[&TestCase], stdout: TimeoutChildStdout<'_>, timer: Option<Timer> ) -> Result<CaselistResult>

Invokes the test suite’s result parser on stdout.

Provided Methods§

source

fn see_more(&self, _name: &str, _caselist_state: &CaselistState) -> String

Optional (recommended) hook for logging on failure to point the user to more information about a specific failure (caselist, log file, etc).

source

fn skips(&self) -> &RegexSet

source

fn flakes(&self) -> &RegexSet

source

fn baseline(&self) -> &RunnerResults

source

fn baseline_status(&self, test: &str) -> Option<RunnerStatus>

source

fn translate_result( &self, result: &TestResult, caselist_state: &CaselistState ) -> RunnerStatus

source

fn skip_test(&self, test: &str) -> bool

source

fn handle_result( &self, _caselist_state: &CaselistState, _result: &TestResult, _status: &RunnerStatus ) -> Result<()>

Used for doing any extra logging that might need to be done for failed tests.

source

fn log_path( &self, caselist_state: &CaselistState, _tests: &[&TestCase] ) -> Result<PathBuf>

source

fn should_save_log( &self, _caselist_state: &CaselistState, _tests: &[&TestCase] ) -> bool

Overrideable hook for whether a log should be always be saved. Sometimes there’s a test that collects general implementation details (API version, extensions) that might be useful for the user to always have, for sanity-checking that the run tested what they intended.

source

fn clean( &self, _caselist_state: &CaselistState, _tests: &[&TestCase], _results: &[RunnerResult] ) -> Result<()>

Hook for cleaning up anything necessary after running a test binary (like removing logs that shouldn’t be saved for all the passing tests).

source

fn handle_exit_status( &self, code: Option<i32>, some_result: Option<&mut TestResult> )

By default, sets the last test result with Crash if the binary crashed after emitting test results. Can also be overridden if the exit status is what determines test success/failure.

source

fn run( &self, caselist_state: &CaselistState, tests: &[&TestCase] ) -> Result<Vec<RunnerResult>>

Invokes the test command on a list of testcases. This is common code because of the trickiness in handling timeouts, whether logs should be saved, etc.

source

fn run_caselist_and_flake_detect( &self, caselist: &[TestCase], caselist_state: &mut CaselistState ) -> Result<Vec<RunnerResult>>

Loop invoking the list of cases, where we run again to check if it was just a flake if an unexpected result occurs.

source

fn process_caselist( &self, tests: Vec<TestCase>, caselist_id: u32 ) -> Result<Vec<RunnerResult>>

source

fn split_tests_to_groups( &self, tests: Vec<TestCase>, tests_per_group: usize, min_tests_per_group: usize, sub_config: &SubRunConfig, include_filters: &[RegexSet] ) -> Result<Vec<(&dyn TestCommand, Vec<TestCase>)>>where Self: Sized,

source

fn caselist_file_path( &self, caselist_state: &CaselistState, suffix: &str ) -> Result<PathBuf>

source

fn prefix(&self) -> &str

Optional prefix to add to the test names, to distinguish between testsuite variations (like enabled debug knobs) in a deqp-runner suite invocation.

Implementors§