int main (int argc, const char * const argv[], char *env[]) { unsigned int i= 0; int totalTests = 0, totalPassed = 0, criticalFailures = 0; int totalDisabled = 0; bool testerTestIncluded = false; std::string testnameRegexp = ""; bool disableNegativeTests = false; bool showall = false; for (int i = 1; i < argc; i++) { std::string arg(argv[i]); if (arg == "--disable-negative-tests") { disableNegativeTests = true; } else if (arg == "--showall") { showall = true; } else if (testnameRegexp.size() == 0) { testnameRegexp = arg; } else { std::cout << "usage: " << argv[0] << " [--showall] [--disable-negative-tests] [regular-expression]\n"; ::exit(1); } } if (testnameRegexp.size() == 0) testnameRegexp = ".*"; // initialize APR apr_status_t result; result = apr_app_initialize(&argc, &argv, 0 /*env*/); if (result) NTA_THROW << "error initializing APR. Err code: " << result; Tester::init(disableNegativeTests); // initialize testinputdir, testoutputdir std::cout << "Executing tests matching the regular expression \"" << testnameRegexp << "\"\n"; // Create each test suite if (regex::match(testnameRegexp, "TesterTest")) { testerTestIncluded = true; TestInfo t(std::string("TesterTest"), new TesterTest); allTesters.push_back(t); // Must be first one } /* * The list of tests is generated automatically by update_app.py, which * creates addtests.hpp. addtests.hpp instantiates all of the test objects * and pushed them on allTesters */ #include "EverythingAddtests.hpp" // Run each test in our test suite std::cout << "Found " << allTesters.size() << " test "; if (allTesters.size() == 1) std::cout << "set\n"; else std::cout << "sets\n"; totalTests = 0; totalPassed = 0; std::vector<std::string> hardfailTests; std::vector< failInfo > criticalfailTests; for(Testers::iterator iter=allTesters.begin(); iter != allTesters.end(); ++iter) { Tester* t = iter->second; t->setName(iter->first); t->runTestsWithExceptionHandling(); totalTests += t->testCount(); totalPassed += t->passCount(); totalDisabled += t->disabledCount(); criticalFailures += (t->criticalFailureOccurred()?1:0); t->report(showall); if (t->testCount() != t->passCount() + t->disabledCount()) { if (t->getName() != "TesterTest") { hardfailTests.push_back(t->getName()); } } if (t->criticalFailureOccurred()) { if (t->getName() != "TesterTest") { failInfo fi(t->getName(), t->getCriticalFailureMsg()); criticalfailTests.push_back(fi); } } } if (testerTestIncluded) { // TesterTests should have exactly 3 failures plus one critical failure. // We consider these as successes if (allTesters[0].second->hardFailCount() == 4) totalPassed += 4; if (allTesters[0].second->criticalFailureOccurred() == true) criticalFailures--; else criticalFailures++; } // Print out high level summary std::cout << "*******************************************************************\n"; std::cout << "* Numenta Unit Test summary\n"; std::cout << "* \n"; std::cout << "* Total categories = " << allTesters.size() << "\n"; std::cout << "* Total tests = " << totalTests << "\n"; std::cout << "* Total passed = " << totalPassed << "\n"; std::cout << "* Total disabled = " << totalDisabled << "\n"; std::cout << "* Total failed = " << totalTests - totalPassed - totalDisabled << "\n"; if (hardfailTests.size() > 0) { for (std::vector<std::string>::const_iterator i = hardfailTests.begin(); i != hardfailTests.end(); i++) { std::cout << " FAILED: " << *i << "\n"; } } std::cout << "* Total critical failures = " << criticalFailures << "\n"; if (criticalfailTests.size() > 0) { for (std::vector< failInfo >::iterator i = criticalfailTests.begin(); i != criticalfailTests.end(); i++) { std::cout << " FAILED: " << i->first << "\n"; std::cout << " MSG: " << i->second << "\n"; } } double testsRun = totalTests - totalDisabled + criticalFailures; if (testsRun > 0) std::cout << "* Success rate = " << (100.0 * ((double)totalPassed / testsRun)) << "\n"; std::cout << "*******************************************************************\n\n\n"; // Clean up for(i=0; i<allTesters.size(); i++) { delete allTesters[i].second; allTesters[i].second = NULL; } // Do not return number of failures. Retval = 256 will be interpreted as exit status 0! int retval; if (totalTests - totalPassed + criticalFailures == 0) { retval = 0; } else { retval = 1; } return retval; }