int runBug27003(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; static const int errnos[] = { 4025, 4026, 4027, 4028, 0 }; int node = res.getRandomNotMasterNodeId(rand()); ndbout_c("node: %d", node); if (res.restartOneDbNode(node, false, true, true)) return NDBT_FAILED; Uint32 pos = 0; for (Uint32 i = 0; i<loops; i++) { while (errnos[pos] != 0) { ndbout_c("Tesing err: %d", errnos[pos]); if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; if (res.insertErrorInNode(node, 1000)) return NDBT_FAILED; if (res.insertErrorInNode(node, errnos[pos])) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node, val2, 2)) return NDBT_FAILED; res.startNodes(&node, 1); NdbSleep_SecSleep(3); pos++; } pos = 0; } if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; res.startNodes(&node, 1); if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug15632(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; int nodeId = restarter.getDbNodeId(1); ndbout << "Restart node " << nodeId << endl; if (restarter.restartOneDbNode(nodeId, /** initial */ false, /** nostart */ true, /** abort */ true)) return NDBT_FAILED; if (restarter.waitNodesNoStart(&nodeId, 1)) return NDBT_FAILED; if (restarter.insertErrorInNode(nodeId, 7165)) return NDBT_FAILED; if (restarter.startNodes(&nodeId, 1)) return NDBT_FAILED; if (restarter.waitNodesStarted(&nodeId, 1)) return NDBT_FAILED; if (restarter.restartOneDbNode(nodeId, /** initial */ false, /** nostart */ true, /** abort */ true)) return NDBT_FAILED; if (restarter.waitNodesNoStart(&nodeId, 1)) return NDBT_FAILED; if (restarter.insertErrorInNode(nodeId, 7171)) return NDBT_FAILED; if (restarter.startNodes(&nodeId, 1)) return NDBT_FAILED; if (restarter.waitNodesStarted(&nodeId, 1)) return NDBT_FAILED; ctx->stopTest(); return NDBT_OK; }
int runBug25554(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; if (restarter.getNumDbNodes() < 4) return NDBT_OK; for (int i = 0; i<loops; i++) { int master = restarter.getMasterNodeId(); int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand()); restarter.restartOneDbNode(node1, false, true, true); int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (restarter.dumpStateOneNode(master, val2, 2)) return NDBT_FAILED; if (restarter.insertErrorInNode(master, 7141)) return NDBT_FAILED; if (restarter.waitNodesNoStart(&node1, 1)) return NDBT_FAILED; if (restarter.dumpStateOneNode(node1, val2, 2)) return NDBT_FAILED; if (restarter.insertErrorInNode(node1, 932)) return NDBT_FAILED; if (restarter.startNodes(&node1, 1)) return NDBT_FAILED; int nodes[] = { master, node1 }; if (restarter.waitNodesNoStart(nodes, 2)) return NDBT_FAILED; if (restarter.startNodes(nodes, 2)) return NDBT_FAILED; if (restarter.waitClusterStarted()) return NDBT_FAILED; } return NDBT_OK; }
int runBug32160(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); NdbRestarter res; if (res.getNumDbNodes() < 2) { return NDBT_OK; } int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); if (res.insertErrorInNode(next, 7194)) { return NDBT_FAILED; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(master, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(master, 7193)) return NDBT_FAILED; int val3[] = { 7099 }; if (res.dumpStateOneNode(master, val3, 1)) return NDBT_FAILED; if (res.waitNodesNoStart(&master, 1)) return NDBT_FAILED; if (res.startNodes(&master, 1)) return NDBT_FAILED; if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug20185(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; HugoOperations hugoOps(*ctx->getTab()); Ndb* pNdb = GETNDB(step); int dump[] = { 7090, 20 } ; if (restarter.dumpStateAllNodes(dump, 2)) return NDBT_FAILED; NdbSleep_MilliSleep(3000); if(hugoOps.startTransaction(pNdb) != 0) return NDBT_FAILED; if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0) return NDBT_FAILED; if (hugoOps.execute_NoCommit(pNdb) != 0) return NDBT_FAILED; int nodeId; const int node = hugoOps.getTransaction()->getConnectedNodeId(); do { nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes()); } while (nodeId == node); if (restarter.insertErrorInAllNodes(7030)) return NDBT_FAILED; if (restarter.insertErrorInNode(nodeId, 7031)) return NDBT_FAILED; NdbSleep_MilliSleep(500); if (hugoOps.execute_Commit(pNdb) == 0) return NDBT_FAILED; NdbSleep_MilliSleep(3000); restarter.waitClusterStarted(); if (restarter.dumpStateAllNodes(dump, 1)) return NDBT_FAILED; return NDBT_OK; }
int runBug57650(NDBT_Context* ctx, NDBT_Step* step) { NdbBackup backup(GETNDB(step)->getNodeId()+1); NdbRestarter res; int node0 = res.getNode(NdbRestarter::NS_RANDOM); res.insertErrorInNode(node0, 5057); unsigned backupId = 0; if (backup.start(backupId) == -1) return NDBT_FAILED; return NDBT_OK; }
int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; HugoOperations hugoOps(*ctx->getTab()); Ndb* pNdb = GETNDB(step); int i = 0; while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){ g_info << i << ": "; int id = i % restarter.getNumDbNodes(); int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; restarter.insertErrorInNode(nodeId, 5041); restarter.insertErrorInAllNodes(8048 + (i & 1)); for(int j = 0; j<records; j++){ if(hugoOps.startTransaction(pNdb) != 0) return NDBT_FAILED; if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0) goto err; int res; if((res = hugoOps.execute_Commit(pNdb)) == 4119) goto done; if(res != 0) goto err; if(hugoOps.closeTransaction(pNdb) != 0) return NDBT_FAILED; } done: if(hugoOps.closeTransaction(pNdb) != 0) return NDBT_FAILED; i++; restarter.waitClusterStarted(60) ; } return result; err: hugoOps.closeTransaction(pNdb); return NDBT_FAILED; }
int runBug16772(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; if (restarter.getNumDbNodes() < 2) { ctx->stopTest(); return NDBT_OK; } int aliveNodeId = restarter.getRandomNotMasterNodeId(rand()); int deadNodeId = aliveNodeId; while (deadNodeId == aliveNodeId) deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes()); if (restarter.insertErrorInNode(aliveNodeId, 930)) return NDBT_FAILED; if (restarter.restartOneDbNode(deadNodeId, /** initial */ false, /** nostart */ true, /** abort */ true)) return NDBT_FAILED; if (restarter.waitNodesNoStart(&deadNodeId, 1)) return NDBT_FAILED; if (restarter.startNodes(&deadNodeId, 1)) return NDBT_FAILED; // It should now be hanging since we throw away NDB_FAILCONF int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10); // So this should fail...i.e it should not reach startphase 3 // Now send a NDB_FAILCONF for deadNo int dump[] = { 7020, 323, 252, 0 }; dump[3] = deadNodeId; if (restarter.dumpStateOneNode(aliveNodeId, dump, 4)) return NDBT_FAILED; if (restarter.waitNodesStarted(&deadNodeId, 1)) return NDBT_FAILED; return ret ? NDBT_OK : NDBT_FAILED; }
int runBug27283(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; if (res.getNumDbNodes() < 2) { return NDBT_OK; } static const int errnos[] = { 7181, 7182, 0 }; Uint32 pos = 0; for (Uint32 i = 0; i<loops; i++) { while (errnos[pos] != 0) { int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); int next2 = res.getNextMasterNodeId(next); int node = (i & 1) ? next : next2; ndbout_c("Tesing err: %d", errnos[pos]); if (res.insertErrorInNode(next, errnos[pos])) return NDBT_FAILED; NdbSleep_SecSleep(3); if (res.waitClusterStarted()) return NDBT_FAILED; pos++; } pos = 0; } return NDBT_OK; }
int runBug26481(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; int node = res.getRandomNotMasterNodeId(rand()); ndbout_c("node: %d", node); if (res.restartOneDbNode(node, true, true, true)) return NDBT_FAILED; if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(node, 7018)) return NDBT_FAILED; if (res.startNodes(&node, 1)) return NDBT_FAILED; res.waitNodesStartPhase(&node, 1, 3); if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; res.startNodes(&node, 1); if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug26457(NDBT_Context* ctx, NDBT_Step* step) { NdbRestarter res; if (res.getNumDbNodes() < 4) return NDBT_OK; int loops = ctx->getNumLoops(); while (loops --) { retry: int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); ndbout_c("master: %d next: %d", master, next); if (res.getNodeGroup(master) == res.getNodeGroup(next)) { res.restartOneDbNode(next, false, false, true); if (res.waitClusterStarted()) return NDBT_FAILED; goto retry; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 }; if (res.dumpStateOneNode(next, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(next, 7180)) return NDBT_FAILED; res.restartOneDbNode(master, false, false, true); if (res.waitClusterStarted()) return NDBT_FAILED; } return NDBT_OK; }
int runBug18414(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; if (restarter.getNumDbNodes() < 2) { ctx->stopTest(); return NDBT_OK; } Ndb* pNdb = GETNDB(step); HugoOperations hugoOps(*ctx->getTab()); HugoTransactions hugoTrans(*ctx->getTab()); int loop = 0; do { if(hugoOps.startTransaction(pNdb) != 0) goto err; if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0) goto err; if(hugoOps.execute_NoCommit(pNdb) != 0) goto err; int node1 = hugoOps.getTransaction()->getConnectedNodeId(); int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand()); if (node1 == -1 || node2 == -1) break; if (loop & 1) { if (restarter.insertErrorInNode(node1, 8050)) goto err; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (restarter.dumpStateOneNode(node2, val2, 2)) goto err; if (restarter.insertErrorInNode(node2, 5003)) goto err; int res= hugoOps.execute_Rollback(pNdb); if (restarter.waitNodesNoStart(&node2, 1) != 0) goto err; if (restarter.insertErrorInAllNodes(0)) goto err; if (restarter.startNodes(&node2, 1) != 0) goto err; if (restarter.waitClusterStarted() != 0) goto err; if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0) goto err; hugoOps.closeTransaction(pNdb); } while(++loop < 5); return NDBT_OK; err: hugoOps.closeTransaction(pNdb); return NDBT_FAILED; }
int runBug28717(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); NdbRestarter res; if (res.getNumDbNodes() < 4) { return NDBT_OK; } int master = res.getMasterNodeId(); int node0 = res.getRandomNodeOtherNodeGroup(master, rand()); int node1 = res.getRandomNodeSameNodeGroup(node0, rand()); ndbout_c("master: %d node0: %d node1: %d", master, node0, node1); if (res.restartOneDbNode(node0, false, true, true)) { return NDBT_FAILED; } { int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 }; NdbLogEventHandle handle = ndb_mgm_create_logevent_handle(res.handle, filter); int dump[] = { DumpStateOrd::DihStartLcpImmediately }; struct ndb_logevent event; for (Uint32 i = 0; i<3; i++) { res.dumpStateOneNode(master, dump, 1); while(ndb_logevent_get_next(handle, &event, 0) >= 0 && event.type != NDB_LE_LocalCheckpointStarted); while(ndb_logevent_get_next(handle, &event, 0) >= 0 && event.type != NDB_LE_LocalCheckpointCompleted); } } if (res.waitNodesNoStart(&node0, 1)) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node0, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(node0, 5010)) return NDBT_FAILED; if (res.insertErrorInNode(node1, 1001)) return NDBT_FAILED; if (res.startNodes(&node0, 1)) return NDBT_FAILED; NdbSleep_SecSleep(3); if (res.insertErrorInNode(node1, 0)) return NDBT_FAILED; if (res.waitNodesNoStart(&node0, 1)) return NDBT_FAILED; if (res.startNodes(&node0, 1)) return NDBT_FAILED; if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int NdbBackup::Fail(NdbRestarter& _restarter, int *Fail_codes, const int sz, bool onMaster){ CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); int nNodes = _restarter.getNumDbNodes(); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = Fail_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::Fail node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; g_info << "waiting some before starting backup" << endl; g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(5); int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); } return NDBT_OK; }
int NdbBackup::NF(NdbRestarter& _restarter, int *NFDuringBackup_codes, const int sz, bool onMaster){ int nNodes = _restarter.getNumDbNodes(); { if(nNodes == 1) return NDBT_OK; int nodeId = _restarter.getMasterNodeId(); CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0, "Could not restart node "<< nodeId); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = NFDuringBackup_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::NF node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0, "failed to set RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; NdbSleep_SecSleep(1); g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); g_info << "number of nodes running " << _restarter.getNumDbNodes() << endl; if (_restarter.getNumDbNodes() != nNodes) { g_err << "Failure: cluster not up" << endl; return NDBT_FAILED; } g_info << "starting new backup" << endl; CHECK(start(backupId) == 0, "failed to start backup"); g_info << "(which should succeed) started with id = " << backupId << endl; g_info << "starting node" << endl; CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); g_info << "node started" << endl; int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(1); } return NDBT_OK; }