/// Returns 0: passed, 1: passed with warnings, -1: failed.
int RunTest(Test &t, int numTimesToRun, int numTrialsPerRun, JSONReport &jsonReport)
{
if (t.runOnlyOnce)
numTimesToRun = numTrialsPerRun = 1;
if (!t.isRandomized)
numTimesToRun = 1;
if (t.isBenchmark)
{
numTimesToRun = numTrialsPerRun = 1;
LOGI_NL("Benchmark '%s': %s", t.name.c_str(), t.description.c_str());
}
else
LOGI_NL("Testing '%s': ", t.name.c_str());
std::vector<tick_t> times;
times.reserve(numTimesToRun);
t.numFails = 0;
t.numPasses = 0;
std::string failReason; // Stores the failure reason of the first failure.
std::vector<std::string> failReasons;
globalTestExpectedToFail = 0;
globalTestFailureDescription = std::string();
for(int j = 0; j < numTimesToRun; ++j)
{
tick_t start = Clock::Tick();
for(int k = 0; k < numTrialsPerRun; ++k)
// for(int k = 0; k < (t.isRandomized ? numTrials : 1); ++k)
{
#ifdef FAIL_USING_EXCEPTIONS
try
{
#endif
t.function(t);
if (globalTestExpectedToFail)
{
globalTestExpectedToFail = 0; // Signal that the following exception reports a failure of this test, and not an expected failure.
throw std::runtime_error(std::string("This test should have failed due to reason '") + globalTestFailureDescription + "', but it didn't fail!");
}
#ifdef FAIL_USING_EXCEPTIONS
}
catch(const TestSkippedException &e)
{
if (failReason.empty())
{
failReason = std::string("SKIPPED: ") + e.what();
LOGW("%s", failReason.c_str());
}
}
catch(const std::exception &e)
{
if (globalTestExpectedToFail)
{
if (globalTestExpectedToFail == 2)
LOGE("This test failed as expected. Caught an exception '%s', failure is due to reason '%s'.", e.what(), globalTestFailureDescription.c_str());
else
LOGI("This test failed as expected. Caught an exception '%s', failure is due to reason '%s'.", e.what(), globalTestFailureDescription.c_str());
}
else
{
if (failReason.empty())
failReason = e.what();
++t.numFails;
}
}
catch(...)
{
++t.numFails;
LOGE("Error: Received an unknown exception type that is _not_ derived from std::exception! This should not happen!");
}
#endif
}
tick_t end = Clock::Tick();
times.push_back(end - start);
}
t.numPasses = numTimesToRun*numTrialsPerRun - t.numFails;
std::sort(times.begin(), times.end());
// Erase outliers. (x% slowest)
const float rateSlowestToDiscard = 0.05f;
int numSlowestToDiscard = (int)(times.size() * rateSlowestToDiscard);
times.erase(times.end() - numSlowestToDiscard, times.end());
tick_t total = 0;
for(size_t j = 0; j < times.size(); ++j)
total += times[j];
if (!t.isBenchmark)
{
if (!times.empty())
{
t.fastestTime = (double)times[0] / numTrialsPerRun;
t.averageTime = (double)total / times.size() / numTrialsPerRun;
t.worstTime = (double)times.back() / numTrialsPerRun;
t.numTimesRun = numTimesToRun;
t.numTrialsPerRun = numTrialsPerRun;
}
else
{
t.fastestTime = t.averageTime = t.worstTime = -1.0;
t.numTimesRun = t.numTrialsPerRun = 0;
}
}
float successRate = (t.numPasses + t.numFails > 0) ? (float)t.numPasses * 100.f / (t.numPasses + t.numFails) : 0.f;
jsonReport.Report(t);
if (t.isBenchmark && t.numFails == 0) // Benchmarks print themselves.
return 0; // 0: Success
int ret = 0; // 0: Success
if (t.numFails == 0)
{
if (t.isRandomized)
LOGI(" ok (%d passes, 100%%)", t.numPasses);
else
LOGI(" ok ");
// ++numTestsPassed;
t.result = TestPassed;
}
else if (successRate >= 95.0f)
{
LOGI_NL(" ok ");
LOGW("Some failures with '%s' (%d passes, %.2f%% of all tries)", failReason.c_str(), t.numPasses, successRate);
// ++numTestsPassed;
// ++numWarnings;
ret = 1; // Success with warnings
t.result = TestPassedWithWarnings;
}
else
{
if (t.isRandomized)
LOGE("FAILED: '%s' (%d passes, %.2f%% of all tries)", failReason.c_str(), t.numPasses, successRate);
else
LOGE("FAILED: '%s'", failReason.c_str());
ret = -1; // Failed
t.result = TestFailed;
}
if (!times.empty())
{
if (t.runOnlyOnce)
LOGI(" Elapsed: %s", FormatTime((double)times[0]).c_str());
else
LOGI(" Fastest: %s, Average: %s, Slowest: %s", FormatTime(t.fastestTime).c_str(), FormatTime(t.averageTime).c_str(), FormatTime(t.worstTime).c_str());
}
return ret;
}
