#include "SDL_config.h"
#include "SDL_test.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#define SDLTEST_INVALID_NAME_FORMAT "(Invalid)"
#define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
#define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n"
static Uint32 SDLTest_TestCaseTimeout = 3600;
char *
SDLTest_GenerateRunSeed(const int length)
{
char *seed = NULL;
SDLTest_RandomContext randomContext;
int counter;
if (length <= 0) {
SDLTest_LogError("The length of the harness seed must be >0.");
return NULL;
}
seed = (char *)SDL_malloc((length + 1) * sizeof(char));
if (seed == NULL) {
SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
SDL_Error(SDL_ENOMEM);
return NULL;
}
SDLTest_RandomInitTime(&randomContext);
for (counter = 0; counter < length; counter++) {
unsigned int number = SDLTest_Random(&randomContext);
char ch = (char) (number % (91 - 48)) + 48;
if (ch >= 58 && ch <= 64) {
ch = 65;
}
seed[counter] = ch;
}
seed[length] = '\0';
return seed;
}
static Uint64
SDLTest_GenerateExecKey(const char *runSeed, const char *suiteName, const char *testName, int iteration)
{
SDLTest_Md5Context md5Context;
Uint64 *keys;
char iterationString[16];
size_t runSeedLength;
size_t suiteNameLength;
size_t testNameLength;
size_t iterationStringLength;
size_t entireStringLength;
char *buffer;
if (runSeed == NULL || runSeed[0] == '\0') {
SDLTest_LogError("Invalid runSeed string.");
return -1;
}
if (suiteName == NULL || suiteName[0] == '\0') {
SDLTest_LogError("Invalid suiteName string.");
return -1;
}
if (testName == NULL || testName[0] == '\0') {
SDLTest_LogError("Invalid testName string.");
return -1;
}
if (iteration <= 0) {
SDLTest_LogError("Invalid iteration count.");
return -1;
}
SDL_memset(iterationString, 0, sizeof(iterationString));
SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
runSeedLength = SDL_strlen(runSeed);
suiteNameLength = SDL_strlen(suiteName);
testNameLength = SDL_strlen(testName);
iterationStringLength = SDL_strlen(iterationString);
entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
buffer = (char *)SDL_malloc(entireStringLength);
if (buffer == NULL) {
SDLTest_LogError("Failed to allocate buffer for execKey generation.");
SDL_Error(SDL_ENOMEM);
return 0;
}
SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
SDLTest_Md5Init(&md5Context);
SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
SDLTest_Md5Final(&md5Context);
SDL_free(buffer);
keys = (Uint64 *)md5Context.digest;
return keys[0];
}
static SDL_TimerID
SDLTest_SetTestTimeout(int timeout, void (SDLCALL *callback)(void))
{
Uint32 timeoutInMilliseconds;
SDL_TimerID timerID;
if (callback == NULL) {
SDLTest_LogError("Timeout callback can't be NULL");
return -1;
}
if (timeout < 0) {
SDLTest_LogError("Timeout value must be bigger than zero.");
return -1;
}
if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
if (SDL_InitSubSystem(SDL_INIT_TIMER)) {
SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
return -1;
}
}
timeoutInMilliseconds = timeout * 1000;
timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
if (timerID == 0) {
SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
return -1;
}
return timerID;
}
#if defined(__WATCOMC__)
#pragma aux SDLTest_BailOut aborts;
#endif
static SDL_NORETURN void SDLCALL
SDLTest_BailOut(void)
{
SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
exit(TEST_ABORTED);
}
static int
SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
{
SDL_TimerID timer = 0;
int testCaseResult = 0;
int testResult = 0;
int fuzzerCount;
if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
{
SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
return TEST_RESULT_SETUP_FAILURE;
}
if (!testCase->enabled && forceTestRun == SDL_FALSE)
{
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
return TEST_RESULT_SKIPPED;
}
SDLTest_FuzzerInit(execKey);
SDLTest_ResetAssertSummary();
timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut);
if (testSuite->testSetUp) {
testSuite->testSetUp(0x0);
if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) {
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
return TEST_RESULT_SETUP_FAILURE;
}
}
testCaseResult = testCase->testCase(0x0);
if (testCaseResult == TEST_SKIPPED) {
testResult = TEST_RESULT_SKIPPED;
} else if (testCaseResult == TEST_STARTED) {
testResult = TEST_RESULT_FAILED;
} else if (testCaseResult == TEST_ABORTED) {
testResult = TEST_RESULT_FAILED;
} else {
testResult = SDLTest_AssertSummaryToTestResult();
}
if (testSuite->testTearDown) {
testSuite->testTearDown(0x0);
}
if (timer) {
SDL_RemoveTimer(timer);
}
fuzzerCount = SDLTest_GetFuzzerInvocationCount();
if (fuzzerCount > 0) {
SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
}
if (testCaseResult == TEST_SKIPPED) {
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
} else if (testCaseResult == TEST_STARTED) {
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
} else if (testCaseResult == TEST_ABORTED) {
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
} else {
SDLTest_LogAssertSummary();
}
return testResult;
}
#if 0#endif
static float GetClock()
{
float currentClock = SDL_GetPerformanceCounter() / (float) SDL_GetPerformanceFrequency();
return currentClock;
}
int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
{
int totalNumberOfTests = 0;
int failedNumberOfTests = 0;
int suiteCounter;
int testCounter;
int iterationCounter;
SDLTest_TestSuiteReference *testSuite;
const SDLTest_TestCaseReference *testCase;
const char *runSeed = NULL;
const char *currentSuiteName;
const char *currentTestName;
Uint64 execKey;
float runStartSeconds;
float suiteStartSeconds;
float testStartSeconds;
float runEndSeconds;
float suiteEndSeconds;
float testEndSeconds;
float runtime;
int suiteFilter = 0;
const char *suiteFilterName = NULL;
int testFilter = 0;
const char *testFilterName = NULL;
SDL_bool forceTestRun = SDL_FALSE;
int testResult = 0;
int runResult = 0;
int totalTestFailedCount = 0;
int totalTestPassedCount = 0;
int totalTestSkippedCount = 0;
int testFailedCount = 0;
int testPassedCount = 0;
int testSkippedCount = 0;
int countSum = 0;
const SDLTest_TestCaseReference **failedTests;
if (testIterations < 1) {
testIterations = 1;
}
if (userRunSeed == NULL || userRunSeed[0] == '\0') {
runSeed = SDLTest_GenerateRunSeed(16);
if (runSeed == NULL) {
SDLTest_LogError("Generating a random seed failed");
return 2;
}
} else {
runSeed = userRunSeed;
}
totalTestFailedCount = 0;
totalTestPassedCount = 0;
totalTestSkippedCount = 0;
runStartSeconds = GetClock();
SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
suiteCounter = 0;
while (testSuites[suiteCounter]) {
testSuite = testSuites[suiteCounter];
suiteCounter++;
testCounter = 0;
while (testSuite->testCases[testCounter])
{
testCounter++;
totalNumberOfTests++;
}
}
if (totalNumberOfTests == 0) {
SDLTest_LogError("No tests to run?");
return -1;
}
failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
if (failedTests == NULL) {
SDLTest_LogError("Unable to allocate cache for failed tests");
SDL_Error(SDL_ENOMEM);
return -1;
}
if (filter != NULL && filter[0] != '\0') {
suiteCounter = 0;
while (testSuites[suiteCounter] && suiteFilter == 0) {
testSuite = testSuites[suiteCounter];
suiteCounter++;
if (testSuite->name != NULL && SDL_strcasecmp(filter, testSuite->name) == 0) {
suiteFilter = 1;
suiteFilterName = testSuite->name;
SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
break;
}
testCounter = 0;
while (testSuite->testCases[testCounter] && testFilter == 0) {
testCase = testSuite->testCases[testCounter];
testCounter++;
if (testCase->name != NULL && SDL_strcasecmp(filter, testCase->name) == 0) {
suiteFilter = 1;
suiteFilterName = testSuite->name;
testFilter = 1;
testFilterName = testCase->name;
SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
break;
}
}
}
if (suiteFilter == 0 && testFilter == 0) {
SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
for (suiteCounter = 0; testSuites[suiteCounter]; ++suiteCounter) {
testSuite = testSuites[suiteCounter];
if (testSuite->name != NULL) {
SDLTest_Log("Test suite: %s", testSuite->name);
}
for (testCounter = 0; testSuite->testCases[testCounter]; ++testCounter) {
testCase = testSuite->testCases[testCounter];
SDLTest_Log(" test: %s", testCase->name);
}
}
SDLTest_Log("Exit code: 2");
SDL_free((void *) failedTests);
return 2;
}
}
suiteCounter = 0;
while(testSuites[suiteCounter]) {
testSuite = testSuites[suiteCounter];
currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
suiteCounter++;
if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
SDL_strcasecmp(suiteFilterName, testSuite->name) != 0) {
SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
suiteCounter,
currentSuiteName);
} else {
testFailedCount = 0;
testPassedCount = 0;
testSkippedCount = 0;
suiteStartSeconds = GetClock();
SDLTest_Log("===== Test Suite %i: '%s' started\n",
suiteCounter,
currentSuiteName);
testCounter = 0;
while(testSuite->testCases[testCounter])
{
testCase = testSuite->testCases[testCounter];
currentTestName = (testCase->name ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
testCounter++;
if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
SDL_strcasecmp(testFilterName, testCase->name) != 0) {
SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
suiteCounter,
testCounter,
currentTestName);
} else {
if (testFilter == 1 && !testCase->enabled) {
SDLTest_Log("Force run of disabled test since test filter was set");
forceTestRun = SDL_TRUE;
}
testStartSeconds = GetClock();
SDLTest_Log("----- Test Case %i.%i: '%s' started",
suiteCounter,
testCounter,
currentTestName);
if (testCase->description != NULL && testCase->description[0] != '\0') {
SDLTest_Log("Test Description: '%s'",
(testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
}
iterationCounter = 0;
while(iterationCounter < testIterations)
{
iterationCounter++;
if (userExecKey != 0) {
execKey = userExecKey;
} else {
execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
}
SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
if (testResult == TEST_RESULT_PASSED) {
testPassedCount++;
totalTestPassedCount++;
} else if (testResult == TEST_RESULT_SKIPPED) {
testSkippedCount++;
totalTestSkippedCount++;
} else {
testFailedCount++;
totalTestFailedCount++;
}
}
testEndSeconds = GetClock();
runtime = testEndSeconds - testStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
if (testIterations > 1) {
SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
} else {
SDLTest_Log("Total Test runtime: %.1f sec", runtime);
}
switch (testResult) {
case TEST_RESULT_PASSED:
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
break;
case TEST_RESULT_FAILED:
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
break;
case TEST_RESULT_NO_ASSERT:
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
break;
}
if (testResult == TEST_RESULT_FAILED) {
failedTests[failedNumberOfTests] = testCase;
failedNumberOfTests++;
}
}
}
suiteEndSeconds = GetClock();
runtime = suiteEndSeconds - suiteStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
countSum = testPassedCount + testFailedCount + testSkippedCount;
if (testFailedCount == 0)
{
SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
}
else
{
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
}
}
}
runEndSeconds = GetClock();
runtime = runEndSeconds - runStartSeconds;
if (runtime < 0.0f) runtime = 0.0f;
SDLTest_Log("Total Run runtime: %.1f sec", runtime);
countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
if (totalTestFailedCount == 0)
{
runResult = 0;
SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
}
else
{
runResult = 1;
SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
}
if (failedNumberOfTests > 0) {
SDLTest_Log("Harness input to repro failures:");
for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
}
}
SDL_free((void *) failedTests);
SDLTest_Log("Exit code: %d", runResult);
return runResult;
}