int stopOnError(F_ARGS){ myRandom48Init((long)NdbTick_CurrentMillisecond()); int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); do { g_info << _restart->m_name << ": node = " << nodeId << endl; CHECK(_restarter.waitClusterStarted(300) == 0, "waitClusterStarted failed"); int val = DumpStateOrd::NdbcntrTestStopOnError; CHECK(_restarter.dumpStateOneNode(nodeId, &val, 1) == 0, "failed to set NdbcntrTestStopOnError"); NdbSleep_SecSleep(3); CHECK(_restarter.waitClusterStarted(300) == 0, "waitClusterStarted failed"); } while (false); return NDBT_OK; }
int getRandomNodeId(NdbRestarter& _restarter) { myRandom48Init((long)NdbTick_CurrentMillisecond()); int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); return nodeId; }
extern "C" void* NdbThreadFuncUpdate(void* pArg) { myRandom48Init((long int)NdbTick_CurrentMillisecond()); unsigned nSucc = 0; unsigned nFail = 0; Ndb* pNdb = NULL ; pNdb = new Ndb("TEST_DB"); VerifyMethodInt(pNdb, init()); VerifyMethodInt(pNdb, waitUntilReady()); while(NdbMutex_Trylock(g_pNdbMutex)) { Uint32 nWarehouse = myRandom48(g_nWarehouseCount); NdbConnection* pNdbConnection = NULL ; VerifyMethodPtr(pNdbConnection, pNdb, startTransaction()); CHK_TR(pNdbConnection) ; // epaulsa NdbOperation* pNdbOperationW = NULL ; VerifyMethodPtr(pNdbOperationW, pNdbConnection, getNdbOperation(c_szWarehouse)); VerifyMethodInt(pNdbOperationW, interpretedUpdateTuple()); VerifyMethodInt(pNdbOperationW, equal(c_szWarehouseNumber, nWarehouse)); VerifyMethodInt(pNdbOperationW, incValue(c_szWarehouseCount, Uint32(1))); Uint32 nWarehouseSum = 0; for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) { NdbOperation* pNdbOperationD = NULL ; VerifyMethodPtr(pNdbOperationD, pNdbConnection, getNdbOperation(c_szDistrict)); VerifyMethodInt(pNdbOperationD, interpretedUpdateTuple()); VerifyMethodInt(pNdbOperationD, equal(c_szDistrictWarehouseNumber, nWarehouse)); VerifyMethodInt(pNdbOperationD, equal(c_szDistrictNumber, nDistrict)); VerifyMethodInt(pNdbOperationD, incValue(c_szDistrictCount, Uint32(1))); Uint32 nDistrictSum = myRandom48(100); nWarehouseSum += nDistrictSum; VerifyMethodInt(pNdbOperationD, setValue(c_szDistrictSum, nDistrictSum)); } VerifyMethodInt(pNdbOperationW, setValue(c_szWarehouseSum, nWarehouseSum)); int iExec = pNdbConnection->execute(Commit); int iError = pNdbConnection->getNdbError().code; if(iExec<0 && iError!=0 && iError!=266 && iError!=626) { ReportMethodInt(iExec, pNdbConnection, "pNdbConnection", "execute(Commit)", __FILE__, __LINE__); } if(iExec==0) { ++nSucc; } else { ++nFail; } VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection)); } ndbout << "update: " << nSucc << " succeeded, " << nFail << " failed " << endl; NdbMutex_Unlock(g_pNdbMutex); delete pNdb; pNdb = NULL ; return NULL; }
int restartRandomNodeInitial(F_ARGS){ myRandom48Init((long)NdbTick_CurrentMillisecond()); int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); g_info << _restart->m_name << ": node = "<<nodeId << endl; CHECK(_restarter.restartOneDbNode(nodeId, true) == 0, "Could not restart node "<<nodeId); return NDBT_OK; }
int restartRandomNodeError(F_ARGS){ myRandom48Init((long)NdbTick_CurrentMillisecond()); int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); ndbout << _restart->m_name << ": node = "<<nodeId << endl; CHECK(_restarter.insertErrorInNode(nodeId, 9999) == 0, "Could not restart node "<<nodeId); return NDBT_OK; }
int twoNodeFailure(F_ARGS){ myRandom48Init((long)NdbTick_CurrentMillisecond()); int randomId = myRandom48(_restarter.getNumDbNodes()); int n[2]; n[0] = _restarter.getDbNodeId(randomId); n[1] = _restarter.getRandomNodeOtherNodeGroup(n[0], rand()); g_info << _restart->m_name << ": node = "<< n[0] << endl; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; CHECK(_restarter.dumpStateOneNode(n[0], val2, 2) == 0, "Failed to dump"); CHECK(_restarter.dumpStateOneNode(n[1], val2, 2) == 0, "Failed to dump"); CHECK(_restarter.insertErrorInNode(n[0], 9999) == 0, "Could not restart node "<< n[0]); // Create random value, max 3 secs int max = 3000; int ms = (myRandom48(max)) + 1; g_info << "Waiting for " << ms << "(" << max << ") ms " << endl; NdbSleep_MilliSleep(ms); g_info << _restart->m_name << ": node = "<< n[1] << endl; CHECK(_restarter.insertErrorInNode(n[1], 9999) == 0, "Could not restart node "<< n[1]); CHECK(_restarter.waitNodesNoStart(n, 2) == 0, "Failed to wait nostart"); _restarter.startNodes(n, 2); return NDBT_OK; }
int main(int argc, const char** argv) { ndb_init(); int verbose = 1; int optind = 0; struct getargs args[1+P_MAX] = { { "verbose", 'v', arg_flag, &verbose, "Print verbose status", "verbose" } }; const int num_args = 1 + P_MAX; int i; for(i = 0; i<P_MAX; i++) { args[i+1].long_name = g_paramters[i].name; args[i+1].short_name = * g_paramters[i].name; args[i+1].type = arg_integer; args[i+1].value = &g_paramters[i].value; BaseString tmp; tmp.assfmt("min: %d max: %d", g_paramters[i].min, g_paramters[i].max); args[i+1].help = strdup(tmp.c_str()); args[i+1].arg_help = 0; } if(getarg(args, num_args, argc, argv, &optind)) { arg_printusage(args, num_args, argv[0], "tabname1 tabname2 ..."); return NDBT_WRONGARGS; } myRandom48Init(NdbTick_CurrentMillisecond()); memset(g_times, 0, sizeof(g_times)); Ndb_cluster_connection con; if(con.connect(12, 5, 1)) { return NDBT_ProgramExit(NDBT_FAILED); } g_ndb = new Ndb(&con, "TEST_DB"); if(g_ndb->init() != 0) { g_err << "init() failed" << endl; goto error; } if(g_ndb->waitUntilReady() != 0) { g_err << "Wait until ready failed" << endl; goto error; } for(i = optind; i<argc; i++) { const char * T = argv[i]; g_info << "Testing " << T << endl; BaseString::snprintf(g_table, sizeof(g_table), T); BaseString::snprintf(g_ordered, sizeof(g_ordered), "IDX_O_%s", T); BaseString::snprintf(g_unique, sizeof(g_unique), "IDX_U_%s", T); if(create_table()) goto error; if(load_table()) goto error; for(int l = 0; l<g_paramters[P_LOOPS].value; l++) { for(int j = 0; j<P_OP_TYPES; j++) { g_paramters[P_OPER].value = j; if(run_read()) goto error; } } print_result(); } if(g_ndb) delete g_ndb; return NDBT_OK; error: if(g_ndb) delete g_ndb; return NDBT_FAILED; }
NdbRestarts(const char* _addr = 0): m_restarter(_addr) { myRandom48Init((long)NdbTick_CurrentMillisecond()); }
int restartNFDuringNR(F_ARGS safety){ myRandom48Init((long)NdbTick_CurrentMillisecond()); int i; const int sz = sizeof(NFDuringNR_codes)/sizeof(NFDuringNR_codes[0]); for(i = 0; i<sz && !ctx->closeToTimeout(safety); i++){ int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); int error = NFDuringNR_codes[i]; g_err << _restart->m_name << ": node = " << nodeId << " error code = " << error << endl; CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0, "Could not restart node "<< nodeId); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 } ; CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0, "failed to set RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); NdbSleep_SecSleep(3); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); CHECK(_restarter.waitNodesStarted(&nodeId, 1) == 0, "waitNodesStarted failed"); } return NDBT_OK; if(_restarter.getNumDbNodes() < 4) return NDBT_OK; #ifdef NDB_USE_GET_ENV char buf[256]; if(NdbEnv_GetEnv("USER", buf, 256) == 0 || strcmp(buf, "ejonore") != 0) return NDBT_OK; for(i = 0; i<sz && !ctx->isTestStopped() && !ctx->closeToTimeout(safety);i++){ const int randomId = myRandom48(_restarter.getNumDbNodes()); int nodeId = _restarter.getDbNodeId(randomId); const int error = NFDuringNR_codes[i]; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int crashNodeId = 0; do { int rand = myRandom48(1000); crashNodeId = _restarter.getRandomNodeOtherNodeGroup(nodeId, rand); } while(crashNodeId == masterNodeId); CHECK(crashNodeId > 0, "getMasterNodeId failed"); g_info << _restart->m_name << " restarting node = " << nodeId << " error code = " << error << " crash node = " << crashNodeId << endl; CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0, "Could not restart node "<< nodeId); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; CHECK(_restarter.dumpStateOneNode(crashNodeId, val, 2) == 0, "failed to set RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(crashNodeId, error) == 0, "failed to set error insert"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); } #endif return NDBT_OK; }
extern "C" void* NdbThreadFuncRead(void* pArg) { myRandom48Init((long int)NdbTick_CurrentMillisecond()); unsigned nSucc = 0; unsigned nFail = 0; NdbRecAttr** ppNdbRecAttrDSum = new NdbRecAttr*[g_nDistrictPerWarehouse]; NdbRecAttr** ppNdbRecAttrDCnt = new NdbRecAttr*[g_nDistrictPerWarehouse]; Ndb* pNdb = NULL ; pNdb = new Ndb("TEST_DB"); VerifyMethodInt(pNdb, init()); VerifyMethodInt(pNdb, waitUntilReady()); while(NdbMutex_Trylock(g_pNdbMutex)) { Uint32 nWarehouse = myRandom48(g_nWarehouseCount); NdbConnection* pNdbConnection = NULL ; VerifyMethodPtr(pNdbConnection, pNdb, startTransaction()); CHK_TR(pNdbConnection) ; // epaulsa NdbOperation* pNdbOperationW = NULL ; VerifyMethodPtr(pNdbOperationW, pNdbConnection, getNdbOperation(c_szWarehouse)); VerifyMethodInt(pNdbOperationW, readTuple()); VerifyMethodInt(pNdbOperationW, equal(c_szWarehouseNumber, nWarehouse)); NdbRecAttr* pNdbRecAttrWSum; VerifyMethodPtr(pNdbRecAttrWSum, pNdbOperationW, getValue(c_szWarehouseSum, 0)); NdbRecAttr* pNdbRecAttrWCnt; VerifyMethodPtr(pNdbRecAttrWCnt, pNdbOperationW, getValue(c_szWarehouseCount, 0)); for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) { NdbOperation* pNdbOperationD = NULL ; VerifyMethodPtr(pNdbOperationD, pNdbConnection, getNdbOperation(c_szDistrict)); VerifyMethodInt(pNdbOperationD, readTuple()); VerifyMethodInt(pNdbOperationD, equal(c_szDistrictWarehouseNumber, nWarehouse)); VerifyMethodInt(pNdbOperationD, equal(c_szDistrictNumber, nDistrict)); VerifyMethodPtr(ppNdbRecAttrDSum[nDistrict], pNdbOperationD, getValue(c_szDistrictSum, 0)); VerifyMethodPtr(ppNdbRecAttrDCnt[nDistrict], pNdbOperationD, getValue(c_szDistrictCount, 0)); } int iExec = pNdbConnection->execute(Commit); int iError = pNdbConnection->getNdbError().code; if(iExec<0 && iError!=0 && iError!=266 && iError!=626) { ReportMethodInt(iExec, pNdbConnection, "pNdbConnection", "execute(Commit)", __FILE__, __LINE__); } if(iExec==0) { Uint32 nSum = 0; Uint32 nCnt = 0; for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) { nSum += ppNdbRecAttrDSum[nDistrict]->u_32_value(); nCnt += ppNdbRecAttrDCnt[nDistrict]->u_32_value(); } if(nSum!=pNdbRecAttrWSum->u_32_value() || nCnt!=g_nDistrictPerWarehouse*pNdbRecAttrWCnt->u_32_value()) { ndbout << "INCONSISTENT!" << endl; ndbout << "iExec==" << iExec << endl; ndbout << "iError==" << iError << endl; ndbout << endl; ndbout << c_szWarehouseSum << "==" << pNdbRecAttrWSum->u_32_value() << ", "; ndbout << c_szWarehouseCount << "==" << pNdbRecAttrWCnt->u_32_value() << endl; ndbout << "nSum==" << nSum << ", nCnt=" << nCnt << endl; for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) { ndbout << c_szDistrictSum << "[" << nDistrict << "]==" << ppNdbRecAttrDSum[nDistrict]->u_32_value() << ", "; ndbout << c_szDistrictCount << "[" << nDistrict << "]==" << ppNdbRecAttrDCnt[nDistrict]->u_32_value() << endl; } VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection)); delete pNdb; pNdb = NULL ; delete[] ppNdbRecAttrDSum; ppNdbRecAttrDSum = NULL ; delete[] ppNdbRecAttrDCnt; ppNdbRecAttrDCnt = NULL ; NDBT_ProgramExit(NDBT_FAILED); } ++nSucc; } else { ++nFail; } VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection)); } ndbout << "read: " << nSucc << " succeeded, " << nFail << " failed " << endl; NdbMutex_Unlock(g_pNdbMutex); delete pNdb; pNdb = NULL ; delete[] ppNdbRecAttrDSum; ppNdbRecAttrDSum = NULL ; delete[] ppNdbRecAttrDCnt; ppNdbRecAttrDCnt = NULL ; return NULL; }
int NdbBackup::Fail(NdbRestarter& _restarter, int *Fail_codes, const int sz, bool onMaster){ CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); int nNodes = _restarter.getNumDbNodes(); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = Fail_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::Fail node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; g_info << "waiting some before starting backup" << endl; g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(5); int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); } return NDBT_OK; }
int NdbBackup::NF(NdbRestarter& _restarter, int *NFDuringBackup_codes, const int sz, bool onMaster){ int nNodes = _restarter.getNumDbNodes(); { if(nNodes == 1) return NDBT_OK; int nodeId = _restarter.getMasterNodeId(); CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0, "Could not restart node "<< nodeId); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = NFDuringBackup_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::NF node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0, "failed to set RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; NdbSleep_SecSleep(1); g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); g_info << "number of nodes running " << _restarter.getNumDbNodes() << endl; if (_restarter.getNumDbNodes() != nNodes) { g_err << "Failure: cluster not up" << endl; return NDBT_FAILED; } g_info << "starting new backup" << endl; CHECK(start(backupId) == 0, "failed to start backup"); g_info << "(which should succeed) started with id = " << backupId << endl; g_info << "starting node" << endl; CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); g_info << "node started" << endl; int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(1); } return NDBT_OK; }
/*************************************************************** **************************************************************** * P U B L I C F U N C T I O N S C O D E S E C T I O N * **************************************************************** ***************************************************************/ void asyncGenerator(ThreadData *data, int parallellism, int millisSendPoll, int minEventSendPoll, int forceSendPoll) { ThreadData * startUp; GeneratorStatistics *st; double periodStop; double benchTimeStart; double benchTimeEnd; int i, j, done; myRandom48Init(data->randomSeed); for(i = 0; i<parallellism; i++){ initGeneratorStatistics(&data[i].generator); } startUp = (ThreadData*)malloc(parallellism * sizeof(ThreadData)); memcpy(startUp, data, (parallellism * sizeof(ThreadData))); /*----------------*/ /* warm up period */ /*----------------*/ periodStop = userGetTime() + (double)data[0].warmUpSeconds; while(userGetTime() < periodStop){ doOneTransaction(startUp, parallellism, millisSendPoll, minEventSendPoll, forceSendPoll); } ndbout_c("Waiting for startup to finish"); /** * Wait for all transactions */ done = 0; while(!done){ done = 1; for(i = 0; i<parallellism; i++){ if(startUp[i].runState != Runnable){ done = 0; break; } } if(!done){ startUp[0].pNDB->sendPollNdb(); } } ndbout_c("Benchmark period starts"); /*-------------------------*/ /* normal benchmark period */ /*-------------------------*/ benchTimeStart = userGetTime(); periodStop = benchTimeStart + (double)data[0].testSeconds; while(userGetTime() < periodStop) doOneTransaction(data, parallellism, millisSendPoll, minEventSendPoll, forceSendPoll); benchTimeEnd = userGetTime(); ndbout_c("Benchmark period done"); /** * Wait for all transactions */ done = 0; while(!done){ done = 1; for(i = 0; i<parallellism; i++){ if(data[i].runState != Runnable){ done = 0; break; } } if(!done){ data[0].pNDB->sendPollNdb(); } } /*------------------*/ /* cool down period */ /*------------------*/ periodStop = userGetTime() + (double)data[0].coolDownSeconds; while(userGetTime() < periodStop){ doOneTransaction(startUp, parallellism, millisSendPoll, minEventSendPoll, forceSendPoll); } done = 0; while(!done){ done = 1; for(i = 0; i<parallellism; i++){ if(startUp[i].runState != Runnable){ done = 0; break; } } if(!done){ startUp[0].pNDB->sendPollNdb(); } } /*---------------------------------------------------------*/ /* add the times for all transaction for inner loop timing */ /*---------------------------------------------------------*/ for(j = 0; j<parallellism; j++){ st = &data[j].generator; st->outerLoopTime = benchTimeEnd - benchTimeStart; st->outerTps = getTps(st->totalTransactions, st->outerLoopTime); } /* ndbout_c("maxsize = %d\n",maxsize); */ free(startUp); }