int main (int argc, const char * const argv[], char *env[])
{
unsigned int i= 0;
int totalTests = 0, totalPassed = 0, criticalFailures = 0;
int totalDisabled = 0;
bool testerTestIncluded = false;
std::string testnameRegexp = "";
bool disableNegativeTests = false;
bool showall = false;
for (int i = 1; i < argc; i++)
{
std::string arg(argv[i]);
if (arg == "--disable-negative-tests")
{
disableNegativeTests = true;
} else if (arg == "--showall")
{
showall = true;
} else if (testnameRegexp.size() == 0) {
testnameRegexp = arg;
} else {
std::cout << "usage: " << argv[0] << " [--showall] [--disable-negative-tests] [regular-expression]\n";
::exit(1);
}
}
if (testnameRegexp.size() == 0)
testnameRegexp = ".*";
// initialize APR
apr_status_t result;
result = apr_app_initialize(&argc, &argv, 0 /*env*/);
if (result)
NTA_THROW << "error initializing APR. Err code: " << result;
Tester::init(disableNegativeTests); // initialize testinputdir, testoutputdir
std::cout << "Executing tests matching the regular expression \""
<< testnameRegexp << "\"\n";
// Create each test suite
if (regex::match(testnameRegexp, "TesterTest")) {
testerTestIncluded = true;
TestInfo t(std::string("TesterTest"), new TesterTest);
allTesters.push_back(t); // Must be first one
}
/*
* The list of tests is generated automatically by update_app.py, which
* creates addtests.hpp. addtests.hpp instantiates all of the test objects
* and pushed them on allTesters
*/
#include "EverythingAddtests.hpp"
// Run each test in our test suite
std::cout << "Found " << allTesters.size() << " test ";
if (allTesters.size() == 1)
std::cout << "set\n";
else
std::cout << "sets\n";
totalTests = 0;
totalPassed = 0;
std::vector<std::string> hardfailTests;
std::vector< failInfo > criticalfailTests;
for(Testers::iterator iter=allTesters.begin();
iter != allTesters.end(); ++iter)
{
Tester* t = iter->second;
t->setName(iter->first);
t->runTestsWithExceptionHandling();
totalTests += t->testCount();
totalPassed += t->passCount();
totalDisabled += t->disabledCount();
criticalFailures += (t->criticalFailureOccurred()?1:0);
t->report(showall);
if (t->testCount() != t->passCount() + t->disabledCount()) {
if (t->getName() != "TesterTest") {
hardfailTests.push_back(t->getName());
}
}
if (t->criticalFailureOccurred()) {
if (t->getName() != "TesterTest") {
failInfo fi(t->getName(), t->getCriticalFailureMsg());
criticalfailTests.push_back(fi);
}
}
}
if (testerTestIncluded) {
// TesterTests should have exactly 3 failures plus one critical failure.
// We consider these as successes
if (allTesters[0].second->hardFailCount() == 4) totalPassed += 4;
if (allTesters[0].second->criticalFailureOccurred() == true)
criticalFailures--;
else criticalFailures++;
}
// Print out high level summary
std::cout << "*******************************************************************\n";
std::cout << "* Numenta Unit Test summary\n";
std::cout << "* \n";
std::cout << "* Total categories = " << allTesters.size() << "\n";
std::cout << "* Total tests = " << totalTests << "\n";
std::cout << "* Total passed = " << totalPassed << "\n";
std::cout << "* Total disabled = " << totalDisabled << "\n";
std::cout << "* Total failed = " << totalTests - totalPassed - totalDisabled << "\n";
if (hardfailTests.size() > 0) {
for (std::vector<std::string>::const_iterator i = hardfailTests.begin();
i != hardfailTests.end(); i++) {
std::cout << " FAILED: " << *i << "\n";
}
}
std::cout << "* Total critical failures = " << criticalFailures << "\n";
if (criticalfailTests.size() > 0) {
for (std::vector< failInfo >::iterator i = criticalfailTests.begin();
i != criticalfailTests.end();
i++) {
std::cout << " FAILED: " << i->first << "\n";
std::cout << " MSG: " << i->second << "\n";
}
}
double testsRun = totalTests - totalDisabled + criticalFailures;
if (testsRun > 0)
std::cout << "* Success rate = " << (100.0 * ((double)totalPassed / testsRun)) << "\n";
std::cout << "*******************************************************************\n\n\n";
// Clean up
for(i=0; i<allTesters.size(); i++)
{
delete allTesters[i].second;
allTesters[i].second = NULL;
}
// Do not return number of failures. Retval = 256 will be interpreted as exit status 0!
int retval;
if (totalTests - totalPassed + criticalFailures == 0) {
retval = 0;
} else {
retval = 1;
}
return retval;
}
