ReturnCode TestCaseLoader::loadAllTestCases()
{
ReturnCode retCode;
if(tcList.size() == 0)
tcList.reserve(20);
if(tcBuilders.size() == 0) {
DATA_INFO("No Test Case Builder Registered :: Operation Failed");
retCode.code = ERROR;
retCode.desc = "No Test Case Builder Registered :: Operation Failed";
return retCode;
}
vector<TestCaseBuilder*>::iterator itrVector;
for(itrVector = tcBuilders.begin(); itrVector != tcBuilders.end(); ++itrVector) {
TestCase* ptrToTC = (*itrVector)->buildTestCase();
DATA_INFO("Test Case built successfully");
int idx = (int)(ptrToTC->getTestCaseNumber());
itrList = tcList.begin();
std::advance(itrList,idx);
tcList.insert(itrList, ptrToTC);
DATA_INFO("Test Case inserted successfully");
}
retCode.code = SUCCESS;
retCode.desc = "All Test Cases loaded";
return retCode;
}
bool run_test(TestCase& testCase, bool catch_exceptions)
{
gCurrentTestCase = testCase;
if (catch_exceptions) {
try {
gCurrentTestCase = testCase;
testCase.execute();
// the test code may declare itself failed
bool result = !gCurrentTestCase.failed;
post_test_sanity_check();
return result;
}
catch (std::runtime_error const& err) {
std::cout << "Error white running test case " << testCase.name << std::endl;
std::cout << err.what() << std::endl;
return false;
}
} else {
testCase.execute();
}
post_test_sanity_check();
return !gCurrentTestCase.failed;
}
Result * Suite::run(){
setupTests();
Result * r = new Result( _name );
Result::cur = r;
for ( list<TestCase*>::iterator i=_tests.begin(); i!=_tests.end(); i++ ){
TestCase * tc = *i;
r->_tests++;
bool passes = false;
log(1) << "\t" << tc->getName() << endl;
try {
tc->run();
passes = true;
}
catch ( ... ){
log() << "unknown exception in test: " << tc->getName() << endl;
}
if ( ! passes )
r->_fails++;
}
return r;
}
// This method is cloned from Operator, just resetting sign and exception bits
// (because we don't have any exception support in this toy example)
TestCase* FPSumOf3Squares::buildRandomTestCase(int i){
TestCase *tc;
/* Generate test cases using random input numbers */
tc = new TestCase(this); // TODO free all this memory when exiting TestBench
/* Fill inputs */
for (unsigned int j = 0; j < ioList_.size(); j++) {
Signal* s = ioList_[j];
if (s->type() == Signal::in) {
// ****** Modification: positive normal numbers with small exponents
mpz_class m = getLargeRandom(wF);
mpz_class bias = (mpz_class(1)<<(wE-1)) - 1;
mpz_class e = getLargeRandom(wE-2) - (mpz_class(1)<<(wE-3)) + bias; // ensure no overflow
mpz_class a = (mpz_class(1)<<(wE+wF+1)) // 01 to denote a normal number
+ (e<<wF) + m;
tc->addInput(s->getName(), a);
}
}
/* Get correct outputs */
emulate(tc);
// cout << tc->getInputVHDL();
// cout << tc->getExpectedOutputVHDL();
return tc;
}
void runTestCase(void* item, void* extraData) {
TestCase testCase = (TestCase)item;
TestSuite testSuite = (TestSuite)extraData;
int result;
if(!testSuite->onlyPrintFailing) {
printTestName(testCase->name);
}
if(testCase->testCaseFunc != NULL) {
if(testSuite->setup != NULL) {
testSuite->setup();
}
result = testCase->testCaseFunc();
if(result == 0) {
if(!testSuite->onlyPrintFailing) {
printTestSuccess();
}
testSuite->numSuccess++;
}
else {
printTestFail();
testSuite->numFail++;
}
if(testSuite->teardown != NULL) {
testSuite->teardown();
}
}
else {
if(!testSuite->onlyPrintFailing) {
_printTestSkipped();
}
testSuite->numSkips++;
}
}
/**
* \brief Executes a Z test for the "German students test" example on the slides.
*/
void StatisticalTesting::germanStudentsTest() {
const double testResults[] = {
97, 77, 100, 99, 100, 75, 76, 95, 96, 90,
96, 70, 71, 98, 97, 97, 67, 100, 97, 100,
92, 130, 100, 100, 95, 100, 92, 94, 89,
89, 82, 65, 100, 98, 85, 100, 93, 87, 100,
97, 73, 100, 93, 110, 95, 110, 79, 92, 96,
100, 87, 92, 110, 110, 100
};
const double distributionMean = 100;
const double distributionStandardDeviation = 12;
const double confidenceLevel = 0.95;
TestCase test;
Hypothesis a("Bonn students are better than other students", GREATER);
Hypothesis b("Bonn students are worse than other students", LESS);
Hypothesis c("Bonn students are at least as good as other students", AT_LEAST);
Hypothesis d("Bonn students are at most as good as other students", AT_MOST);
Hypothesis e("Bonn students are as good as other students", EQUAL);
Hypothesis f("Bonn students perform differently from other students", DIFFERENT);
/* TODO: Select a hypothesis and a null hypothesis from the above choices a-f and
* call test.setHypothesis(...) and test.setNullHypothesis(...) with the chosen
* variable.
*/
test.setHypothesis(b);
test.setNullHypothesis(c);
oneSampleZTest(arrayToVector(testResults), distributionMean, distributionStandardDeviation, confidenceLevel, test);
}
static int simple_unittest_suite_run(Suite* suite)
{
printf(" Unit test for Suite : %s \n", suite->name);
int fail_count = 0;
TestCase* index = suite->cases;
while (index != NULL)
{
if (suite->setup_func)
{
suite->setup_func();
}
simple_unittest_set_flag(1);
testInst->cur_time = simple_monotonic_time_now();
index->func();
int64_t tmp = simple_monotonic_time_now();
if (ut_flag == 1)
{
if(isatty(stdout->_fileno) == 1){
printf("\e[32m PASS \e[37m\x1b[0m , %s , %"PRId64" micro-seconds\n",
index->name, (tmp - testInst->cur_time));
}
else{
printf(" PASS , %s , %"PRId64" micro-seconds\n",
index->name, (tmp - testInst->cur_time));
}
testInst->cur_time = tmp;
}
void SummaryTree::addTestCases()
{
QTreeWidgetItem *curItem = currentItem();
if (indexOfTopLevelItem(curItem) == -1)
curItem = curItem->parent();
int index = indexOfTopLevelItem(curItem);
Task *curTask = curContest->getTask(index);
AddTestCasesWizard *wizard = new AddTestCasesWizard(this);
wizard->setSettings(settings, curTask->getTaskType() == Task::Traditional);
if (wizard->exec() == QDialog::Accepted) {
QList<QStringList> inputFiles = wizard->getMatchedInputFiles();
QList<QStringList> outputFiles = wizard->getMatchedOutputFiles();
for (int i = 0; i < inputFiles.size(); i ++) {
addTestCase();
QTreeWidgetItem *curItem = currentItem();
QTreeWidgetItem *parentItem = curItem->parent();
int taskIndex = indexOfTopLevelItem(parentItem);
int testCaseIndex = parentItem->indexOfChild(curItem);
Task *curTask = curContest->getTask(taskIndex);
TestCase *curTestCase = curTask->getTestCase(testCaseIndex);
curTestCase->setFullScore(wizard->getFullScore());
curTestCase->setTimeLimit(wizard->getTimeLimit());
curTestCase->setMemoryLimit(wizard->getMemoryLimit());
for (int j = 0; j < inputFiles[i].size(); j ++)
curTestCase->addSingleCase(inputFiles[i][j], outputFiles[i][j]);
setCurrentItem(parentItem);
setCurrentItem(curItem);
}
}
delete wizard;
}
Result runTestCase (const TestCase& testCase)
{
Result result;
printf ("%s\n", testCase.getName ().c_str());
intend++;
for (auto& it : testCase)
{
try {
if (testCase.setup ())
{
testCase.setup () (this);
}
if (runTest (it.first, it.second))
{
result.succeded++;
}
else
{
result.failed++;
}
if (testCase.teardown ())
{
testCase.teardown () (this);
}
} catch (const std::exception& exc)
{
result.failed++;
}
}
intend--;
return result;
}
/**
* \brief Executes a two sample T test for the "Planning" example on the exercise sheet.
*/
void StatisticalTesting::planning() {
const double confidenceLevel = 0.95;
const double myPlanner[] = {
90, 104, 142, 143, 121,
190, 92, 93, 166, 110,
191, 122, 129, 176, 110,
45, 78, 166, 173, 115,
197, 63, 156, 124, 98
};
const double baselinePlanner[] = {
56, 92, 145, 117, 121,
91, 147, 174, 122, 111,
143, 142, 189, 129, 92,
112, 122, 120, 125, 200,
137, 147, 89, 101, 108
};
TestCase test;
Hypothesis a("My planner produces longer paths than the baseline.", GREATER);
Hypothesis b("My planner produces shorter paths than the baseline.", LESS);
Hypothesis c("My plans are at most as long as the baseline plans.", AT_MOST);
Hypothesis d("My plans are at least as long as the baseline plans.", AT_LEAST);
Hypothesis e("My plans are as long as the baseline paths.", EQUAL);
Hypothesis f("My plans have different lengths than the baseline paths.", DIFFERENT);
test.setHypothesis(f);
test.setNullHypothesis(e);
twoSampleTTest(arrayToVector(myPlanner), arrayToVector(baselinePlanner),
confidenceLevel, test);
}
ReturnCode TestCaseLoader::loadTestCase(unsigned int tcIdx)
{
// TODO : to be reviewed : segmentation fault without the initial PAD
ReturnCode retCode;
if(tcList.size() == 0) {
tcList.reserve((tcIdx + 10));
TestCase* pad = new TestCase();
pad->setTestCaseNumber(100000);
pad->setSetupTestItem(0);
pad->setTearDownTestItem(0);
pad->addRunnableTestItem(0);
tcList.push_back(pad);
}
if(tcIdx > tcBuilders.size()) {
DATA_INFO("Index Out of Bound :: Operation Failed");
retCode.code = ERROR;
retCode.desc = "Index Out of Bound :: Operation Failed";
return retCode;
}
TestCaseBuilder* tcBuild = tcBuilders[tcIdx];
TestCase* ptrToTC = tcBuild->buildTestCase();
DATA_INFO("Test Case built successfully");
itrList = tcList.begin();
std::advance(itrList,tcIdx);
tcList.insert(itrList, ptrToTC);
DATA_INFO("Test Case inserted successfully");
retCode.code = SUCCESS;
retCode.desc = "Test Case loaded";
return retCode;
}
virtual void registerTest( TestCase const& testCase ) {
std::string name = testCase.getTestCaseInfo().name;
if( name == "" ) {
std::ostringstream oss;
oss << "Anonymous test case " << ++m_unnamedCount;
return registerTest( testCase.withName( oss.str() ) );
}
if( m_functions.find( testCase ) == m_functions.end() ) {
m_functions.insert( testCase );
m_functionsInOrder.push_back( testCase );
if( !testCase.isHidden() )
m_nonHiddenFunctions.push_back( testCase );
}
else {
TestCase const& prev = *m_functions.find( testCase );
{
Colour colourGuard( Colour::Red );
Catch::cerr() << "error: TEST_CASE( \"" << name << "\" ) already defined.\n"
<< "\tFirst seen at " << prev.getTestCaseInfo().lineInfo << "\n"
<< "\tRedefined at " << testCase.getTestCaseInfo().lineInfo << std::endl;
}
exit(1);
}
}
TEST_P(EncodeTest, EncodesAsExpected) {
std::string id = GetParam().first;
std::cout << "Test ID: " << id << std::endl;
const int passLength = 128;
char passwd[passLength];
char * const password = passwd;
char keyIDBuf[passLength];
char * const keyID = keyIDBuf;
TestCase testCase = GetParam().second;
char const * const userName = testCase.at("fullName").c_str();
char const * const masterPassword = testCase.at("masterPassword").c_str();
char const * const siteTypeString = testCase.at("siteType").c_str();
char const * const siteName = testCase.at("siteName").c_str();
const uint32_t siteCounter = (uint32_t)atoll(testCase.at("siteCounter").c_str());
char const * const siteScope = ScopeForVariant(VariantWithName(testCase.at("siteVariant").c_str()));
char const * const siteContext = testCase.at("siteContext").c_str();
int bOK = mpw_core(siteScope, password, passLength, userName, masterPassword, siteTypeString, siteName, siteCounter, siteContext, keyID, passLength);
EXPECT_EQ(0, bOK);
EXPECT_EQ(std::string(testCase.at("keyID")), std::string(keyID));
EXPECT_EQ(std::string(testCase.at("result")), std::string(password));
}
void
Runner::runTestAt( unsigned int index, TestResult &result ) const
{
TestCase *test = tests_[index]();
result.setTestName( test->testName() );
printf( "Testing %s: ", test->testName() );
fflush( stdout );
#if JSON_USE_EXCEPTION
try
{
#endif // if JSON_USE_EXCEPTION
test->run( result );
#if JSON_USE_EXCEPTION
}
catch ( const std::exception &e )
{
result.addFailure( __FILE__, __LINE__,
"Unexpected exception caugth:" ) << e.what();
}
#endif // if JSON_USE_EXCEPTION
delete test;
const char *status = result.failed() ? "FAILED"
: "OK";
printf( "%s\n", status );
fflush( stdout );
}
int test::runAllTests() {
const std::vector<TestCase*> &testCases = TestCase::getCases();
int passed = 0;
const char *prevFixture = "";
for (int i = 0; i < testCases.size(); i++) {
TestCase *tc = testCases[i];
if (strcmp(tc->fixture, prevFixture) != 0) {
if (strlen(prevFixture) != 0) {
cout << endl;
}
cout << tc->fixture << ":" << endl;
prevFixture = tc->fixture;
}
cout << "- " << tc->name << ": " << flush;
try {
tc->run();
cout << "[OK]" << endl;
passed++;
} catch (TestFailedException &e) {
cout << "[FAILED]" << endl;
cout << " " << e.message << endl;
cout << " (" << e.file << ":" << e.line << ")" << endl;
}
}
cout << endl << passed << " / " << testCases.size() << " tests passed." << endl;
return (passed == testCases.size() ? 0 : 1);
}
std::string
Runner::testNameAt( unsigned int index ) const
{
TestCase *test = tests_[index]();
std::string name = test->testName();
delete test;
return name;
}
TestSuite::~TestSuite() {
unsigned i;
for (i = 0; i < this->mTests->count(); ++i) {
TestCase * ts = (TestCase*)this->mTests->objectAtIndex(i);
ts->release();
}
this->mTests->release();
}
Result * Suite::run(){
tlogLevel = -1;
log(1) << "\t about to setupTests" << endl;
setupTests();
log(1) << "\t done setupTests" << endl;
Result * r = new Result( _name );
Result::cur = r;
/* see note in SavedContext */
//writelock lk("");
for ( list<TestCase*>::iterator i=_tests.begin(); i!=_tests.end(); i++ ){
TestCase * tc = *i;
r->_tests++;
bool passes = false;
log(1) << "\t going to run test: " << tc->getName() << endl;
stringstream err;
err << tc->getName() << "\t";
try {
tc->run();
passes = true;
}
catch ( MyAssertionException * ae ){
err << ae->ss.str();
delete( ae );
}
catch ( std::exception& e ){
err << " exception: " << e.what();
}
catch ( int x ){
err << " caught int : " << x << endl;
}
catch ( ... ){
cerr << "unknown exception in test: " << tc->getName() << endl;
}
if ( ! passes ){
string s = err.str();
log() << "FAIL: " << s << endl;
r->_fails++;
r->_messages.push_back( s );
}
}
if ( r->_fails )
r->_rc = 17;
log(1) << "\t DONE running tests" << endl;
return r;
}
void TestRegistry::registerTest( TestCase const& testCase ) {
std::string name = testCase.getTestCaseInfo().name;
if( name.empty() ) {
ReusableStringStream rss;
rss << "Anonymous test case " << ++m_unnamedCount;
return registerTest( testCase.withName( rss.str() ) );
}
m_functions.push_back( testCase );
}
void SDKUnitTestListener::OnTestCaseEnd(const TestCase& test_case) {
if (!GTEST_FLAG(print_time)) return;
const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests");
PushResult( "[----------] " );
PushResult( UTIL_VarArgs("%s from %s (%s ms total)\n\n",
counts.c_str(), test_case.name(),
internal::StreamableToString(test_case.elapsed_time()).c_str() ) );
}
virtual void registerTest( TestCase const& testCase ) {
std::string name = testCase.getTestCaseInfo().name;
if( name.empty() ) {
std::ostringstream oss;
oss << "Anonymous test case " << ++m_unnamedCount;
return registerTest( testCase.withName( oss.str() ) );
}
m_functions.push_back( testCase );
}
TestSuite::~TestSuite()
{
list<TestCase *>::iterator iter;
iter = this->tests.begin();
while (iter != this->tests.end()) {
TestCase *tc = *iter;
tc->releaseReference();
iter++;
}
}
void
run_h_tc_child(void* v)
{
run_h_tc_data* data = static_cast< run_h_tc_data* >(v);
TestCase tc;
tc.init(data->m_config);
tc.run("result");
std::exit(EXIT_SUCCESS);
}
void TestSuite::run() {
unsigned i;
this->mAll = 0;
this->mPassed = 0;
for (i = 0; i < this->mTests->count(); ++i) {
TestCase * ts = (TestCase*)this->mTests->objectAtIndex(i);
this->mPassed += ts->run();
this->mAll += ts->getNumTest();
}
}
// Called before the test case starts.
void TersePrinter::OnTestCaseStart(const TestCase& test_case)
{
if (useTerseOutput) return;
ColoredPrintf(COLOR_GREEN, "[----------] ");
printf("%s from %s\n",
FormatTestCount(test_case.test_to_run_count()).c_str(),
test_case.name());
fflush(stdout);
}
int main(int argc, char* argv[])
{
int T;
while(true) {
TestCase t;
if (!t.dotest())
break;
}
return 0;
}
void TestListView::onListTestsFinished(void) {
// sanity check
Q_ASSERT_X(m_runner, Q_FUNC_INFO, "null test runner");
if ( m_runner == 0 )
return;
// clear any previous data (necessary here?)
clear();
// foreach program
const int numPrograms = m_runner->programCount();
for ( int i = 0; i < numPrograms; ++i ) {
TestProgram* program = m_runner->programAt(i);
Q_ASSERT_X(program, Q_FUNC_INFO, "null test program");
// create top-level item for program
QTreeWidgetItem* programItem = new QTreeWidgetItem;
programItem->setData(0, Qt::DisplayRole, program->programName());
programItem->setData(0, Qt::UserRole, QVariant::fromValue(program));
// programItem->setCheckState(0, Qt::Checked);
// foreach test suite
const int numSuites = program->suiteCount();
for ( int j = 0; j < numSuites; ++j ) {
TestSuite* suite = program->suiteAt(j);
Q_ASSERT_X(suite, Q_FUNC_INFO, "null test suite");
// create item for suite
QTreeWidgetItem* suiteItem = new QTreeWidgetItem(programItem);
suiteItem->setData(0, Qt::DisplayRole, suite->name());
suiteItem->setData(0, Qt::UserRole, QVariant::fromValue(suite));
// suiteItem->setCheckState(0, Qt::Checked);
// foreach test case
const int numTests = suite->testCount();
for ( int k = 0; k < numTests; ++k ) {
TestCase* test = suite->testAt(k);
Q_ASSERT_X(test, Q_FUNC_INFO, "null test case");
// create item for test
QTreeWidgetItem* testItem = new QTreeWidgetItem(suiteItem);
testItem->setData(0, Qt::DisplayRole, test->name());
testItem->setData(0, Qt::UserRole, QVariant::fromValue(test));
// testItem->setCheckState(0, Qt::Checked);
}
}
// add program item to table
addTopLevelItem(programItem);
}
// start with all items collapsed
collapseAll();
}
int TestEnv::RunProcessTestCase(TestCase& obj, void(TestCase::*meth)(), int timeout)
{
TestCaseCall call(obj, meth);
TestEnv::in_child_process = true;
HANDLE thread = (HANDLE)_beginthread( ThreadProc, 0, &call );
if (thread == NULL)
{
REPORT_ERROR("TestEnv::RunProcessTestCase: failed to start a test case thread");
}
// make sure to restore main process's signal handlers before re-throwing an exception
DWORD rc=0;
DWORD result=WaitForSingleObject( (HANDLE)thread, timeout == 0 ? INFINITE : timeout*1000);
try
{
switch (result)
{
case WAIT_OBJECT_0:
if (GetExitCodeThread(thread, &rc) == 0)
REPORT_ERROR("RunProcessTestCase failed");
break;
case WAIT_TIMEOUT:
if (!CloseHandle(thread))
REPORT_ERROR("CloseHandle failed");
cerr << "child process timed out" << endl;
rc=TEST_CASE_TIMED_OUT;
break;
default:
REPORT_ERROR("WaitForSingleObject failed");
break;
}
}
catch (const exception& ex)
{
REPORT_ERROR(obj.GetName() + " threw " + ex.what());
rc=TEST_CASE_FAILED;
}
catch (const ncbi::NK::execution_aborted&)
{
REPORT_ERROR(obj.GetName() + " aborted ");
rc=TEST_CASE_FAILED;
}
catch (...)
{
REPORT_ERROR(obj.GetName() + " threw something ");
rc=TEST_CASE_FAILED;
set_handlers();
throw;
}
#undef CALL_FAILED
set_handlers();
in_child_process = false;
return (int)rc;
}
void SDKUnitTestListener::OnTestCaseStart(const TestCase& test_case) {
const std::string counts =
FormatCountableNoun(test_case.test_to_run_count(), "test", "tests");
PushResult( "[----------] " );
PushResult( UTIL_VarArgs("%s from %s", counts.c_str(), test_case.name()) );
if (test_case.type_param() == NULL) {
PushResult( "\n" );
} else {
PushResult( UTIL_VarArgs(", where %s = %s\n", kTypeParamLabel, test_case.type_param()) );
}
}
const TestCase* TestPlan::retrieveTestCase(unsigned int tcIdx) const
{
if(tcIdx > listOfTests.size()) {
DATA_ERR_VAL("Povided Index is out of bound, current Count#", listOfTests.size());
return 0;
}
TestCase* tmp = listOfTests.at(tcIdx);
DATA_INFO_VAL("Test Case retrievable successfully, Number#", tmp->getTestCaseNumber());
//
return tmp;
}
