int main(int argc, char **argv) { TestData::InitTestData(); int numTotalRuns = (argc >= 2) ? atoi(argv[1]) : 100; int numTrialsPerTimedBlock = (argc >= 3) ? atoi(argv[2]) : 100; #ifdef EMSCRIPTEN numTotalRuns = numTrialsPerTimedBlock = 10; #endif // A list of test prefixes to include in the run. std::vector<const char *>prefixesArray; for(int i = 3; i < argc; ++i) { if (argv[i][0] != '-' && argv[i][0] != '/') prefixesArray.push_back(argv[i]); } if (prefixesArray.empty()) prefixesArray.push_back(""); // Empty prefix runs all tests. prefixesArray.push_back(0); // Sentinel to terminate prefix string list. const char * const *prefixes = &prefixesArray[0]; if (numTotalRuns == 0 || numTrialsPerTimedBlock == 0) { LOGI("Usage: %s <numTotalRuns> <numTrialsPerTimedBlock>", argv[0]); LOGI(" Runs all tests."); LOGI(" %s <numTotalRuns> <numTrialsPerTimedBlock> prefix1 prefix2 prefix3...", argv[0]); LOGI(" Runs all tests starting with one of the given prefixes, or residing in one of the named code files."); return 0; } JSONReport jsonReport; std::string jsonFilename = "test_results.json"; for(int i = 1; i+1 < argc; ++i) if (!strcmp(argv[i], "--json")) jsonFilename = argv[i+1]; // Allow overriding the output file name from command line. jsonReport.Create(jsonFilename.c_str()); int numFailures = RunTests(numTotalRuns, numTrialsPerTimedBlock, prefixes, jsonReport); LOGI("%d", globalPokedData); // When --exit0 is passed, we forcibly return 0 and not the number of failed tests. // Used by buildbot in valgrind runs to ignore any failures - the failures are detected // in a "real" run instead that carry more randomized trial runs. for(int i = 1; i < argc; ++i) if (!strcmp(argv[i], "--exit0")) return 0; return numFailures; // exit code of 0 denotes a successful run. }
int main(int argc, char **argv) { int numTotalRuns = (argc >= 2) ? atoi(argv[1]) : 100; int numTrialsPerTimedBlock = (argc >= 3) ? atoi(argv[2]) : 100; #ifdef EMSCRIPTEN numTotalRuns = numTrialsPerTimedBlock = 10; #endif const char * const noPrefixes[] = { "", 0 }; const char * const *prefixes = (argc >= 4) ? &argv[3] : noPrefixes; if (numTotalRuns == 0 || numTrialsPerTimedBlock == 0) { LOGI("Usage: %s <numTotalRuns> <numTrialsPerTimedBlock>", argv[0]); LOGI(" Runs all tests."); LOGI(" %s <numTotalRuns> <numTrialsPerTimedBlock> prefix1 prefix2 prefix3...", argv[0]); LOGI(" Runs all tests starting with one of the given prefixes, or residing in one of the named code files."); return 0; } JSONReport jsonReport; std::string jsonFilename = "test_results.json"; for(int i = 1; i+1 < argc; ++i) if (!strcmp(argv[i], "--json")) jsonFilename = argv[i+1]; // Allow overriding the output file name from command line. jsonReport.Create(jsonFilename.c_str()); int numFailures = RunTests(numTotalRuns, numTrialsPerTimedBlock, prefixes, jsonReport); LOGI("%d", globalPokedData); // When --exit0 is passed, we forcibly return 0 and not the number of failed tests. // Used by buildbot in valgrind runs to ignore any failures - the failures are detected // in a "real" run instead that carry more randomized trial runs. for(int i = 1; i < argc; ++i) if (!strcmp(argv[i], "--exit0")) return 0; return numFailures; // exit code of 0 denotes a successful run. }
/// Returns 0: passed, 1: passed with warnings, -1: failed. int RunTest(Test &t, int numTimesToRun, int numTrialsPerRun, JSONReport &jsonReport) { if (t.runOnlyOnce) numTimesToRun = numTrialsPerRun = 1; if (!t.isRandomized) numTimesToRun = 1; if (t.isBenchmark) { numTimesToRun = numTrialsPerRun = 1; LOGI_NL("Benchmark '%s': %s", t.name.c_str(), t.description.c_str()); } else LOGI_NL("Testing '%s': ", t.name.c_str()); std::vector<tick_t> times; times.reserve(numTimesToRun); t.numFails = 0; t.numPasses = 0; std::string failReason; // Stores the failure reason of the first failure. std::vector<std::string> failReasons; globalTestExpectedToFail = 0; globalTestFailureDescription = std::string(); for(int j = 0; j < numTimesToRun; ++j) { tick_t start = Clock::Tick(); for(int k = 0; k < numTrialsPerRun; ++k) // for(int k = 0; k < (t.isRandomized ? numTrials : 1); ++k) { #ifdef FAIL_USING_EXCEPTIONS try { #endif t.function(t); if (globalTestExpectedToFail) { globalTestExpectedToFail = 0; // Signal that the following exception reports a failure of this test, and not an expected failure. throw std::runtime_error(std::string("This test should have failed due to reason '") + globalTestFailureDescription + "', but it didn't fail!"); } #ifdef FAIL_USING_EXCEPTIONS } catch(const TestSkippedException &e) { if (failReason.empty()) { failReason = std::string("SKIPPED: ") + e.what(); LOGW("%s", failReason.c_str()); } } catch(const std::exception &e) { if (globalTestExpectedToFail) { if (globalTestExpectedToFail == 2) LOGE("This test failed as expected. Caught an exception '%s', failure is due to reason '%s'.", e.what(), globalTestFailureDescription.c_str()); else LOGI("This test failed as expected. Caught an exception '%s', failure is due to reason '%s'.", e.what(), globalTestFailureDescription.c_str()); } else { if (failReason.empty()) failReason = e.what(); ++t.numFails; } } catch(...) { ++t.numFails; LOGE("Error: Received an unknown exception type that is _not_ derived from std::exception! This should not happen!"); } #endif } tick_t end = Clock::Tick(); times.push_back(end - start); } t.numPasses = numTimesToRun*numTrialsPerRun - t.numFails; std::sort(times.begin(), times.end()); // Erase outliers. (x% slowest) const float rateSlowestToDiscard = 0.05f; int numSlowestToDiscard = (int)(times.size() * rateSlowestToDiscard); times.erase(times.end() - numSlowestToDiscard, times.end()); tick_t total = 0; for(size_t j = 0; j < times.size(); ++j) total += times[j]; if (!t.isBenchmark) { if (!times.empty()) { t.fastestTime = (double)times[0] / numTrialsPerRun; t.averageTime = (double)total / times.size() / numTrialsPerRun; t.worstTime = (double)times.back() / numTrialsPerRun; t.numTimesRun = numTimesToRun; t.numTrialsPerRun = numTrialsPerRun; } else { t.fastestTime = t.averageTime = t.worstTime = -1.0; t.numTimesRun = t.numTrialsPerRun = 0; } } float successRate = (t.numPasses + t.numFails > 0) ? (float)t.numPasses * 100.f / (t.numPasses + t.numFails) : 0.f; jsonReport.Report(t); if (t.isBenchmark && t.numFails == 0) // Benchmarks print themselves. return 0; // 0: Success int ret = 0; // 0: Success if (t.numFails == 0) { if (t.isRandomized) LOGI(" ok (%d passes, 100%%)", t.numPasses); else LOGI(" ok "); // ++numTestsPassed; t.result = TestPassed; } else if (successRate >= 95.0f) { LOGI_NL(" ok "); LOGW("Some failures with '%s' (%d passes, %.2f%% of all tries)", failReason.c_str(), t.numPasses, successRate); // ++numTestsPassed; // ++numWarnings; ret = 1; // Success with warnings t.result = TestPassedWithWarnings; } else { if (t.isRandomized) LOGE("FAILED: '%s' (%d passes, %.2f%% of all tries)", failReason.c_str(), t.numPasses, successRate); else LOGE("FAILED: '%s'", failReason.c_str()); ret = -1; // Failed t.result = TestFailed; } if (!times.empty()) { if (t.runOnlyOnce) LOGI(" Elapsed: %s", FormatTime((double)times[0]).c_str()); else LOGI(" Fastest: %s, Average: %s, Slowest: %s", FormatTime(t.fastestTime).c_str(), FormatTime(t.averageTime).c_str(), FormatTime(t.worstTime).c_str()); } return ret; }