int runBug29364(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; Ndb* pNdb = GETNDB(step); HugoTransactions hugoTrans(*ctx->getTab()); if (restarter.getNumDbNodes() < 4) return NDBT_OK; int dump0[] = { 9000, 0 } ; int dump1[] = { 9001, 0 } ; Uint32 ownNode = refToNode(pNdb->getReference()); dump0[1] = ownNode; for (; loops; loops --) { int node0 = restarter.getDbNodeId(rand() % restarter.getNumDbNodes()); int node1 = restarter.getRandomNodeOtherNodeGroup(node0, rand()); restarter.restartOneDbNode(node0, false, true, true); restarter.waitNodesNoStart(&node0, 1); restarter.startNodes(&node0, 1); restarter.waitClusterStarted(); restarter.restartOneDbNode(node1, false, true, true); restarter.waitNodesNoStart(&node1, 1); if (restarter.dumpStateOneNode(node1, dump0, 2)) return NDBT_FAILED; restarter.startNodes(&node1, 1); do { for (Uint32 i = 0; i < 100; i++) { hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead); } } while (restarter.waitClusterStarted(5) != 0); if (restarter.dumpStateOneNode(node1, dump1, 1)) return NDBT_FAILED; } return NDBT_OK; }
int runFail(NDBT_Context* ctx, NDBT_Step* step){ NdbBackup backup(GETNDB(step)->getNodeId()+1); NdbRestarter restarter; if (restarter.getNumDbNodes() < 2){ ctx->stopTest(); return NDBT_OK; } if(restarter.waitClusterStarted(60) != 0){ g_err << "Cluster failed to start" << endl; return NDBT_FAILED; } if (testMaster) { if (testSlave) { if (backup.FailMasterAsSlave(restarter) != NDBT_OK){ return NDBT_FAILED; } } else { if (backup.FailMaster(restarter) != NDBT_OK){ return NDBT_FAILED; } } } else { if (backup.FailSlave(restarter) != NDBT_OK){ return NDBT_FAILED; } } return NDBT_OK; }
int runRestartGciControl(NDBT_Context* ctx, NDBT_Step* step){ int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); UtilTransactions utilTrans(*ctx->getTab()); NdbRestarter restarter; // Wait until we have enough records in db int count = 0; while (count < records){ if (utilTrans.selectCount(pNdb, 64, &count) != 0){ ctx->stopTest(); return NDBT_FAILED; } } // Restart cluster with abort if (restarter.restartAll(false, false, true) != 0){ ctx->stopTest(); return NDBT_FAILED; } // Stop the other thread ctx->stopTest(); if (restarter.waitClusterStarted(300) != 0){ return NDBT_FAILED; } if (pNdb->waitUntilReady() != 0){ return NDBT_FAILED; } return NDBT_OK; }
int runCheckStarted(NDBT_Context* ctx, NDBT_Step* step) { // Check cluster is started NdbRestarter restarter; if(restarter.waitClusterStarted() != 0) { g_err << "All nodes was not started " << endl; return NDBT_FAILED; } // Check atrtclient is started AtrtClient atrt; if(!atrt.waitConnected()) { g_err << "atrt server was not started " << endl; return NDBT_FAILED; } // Make sure atrt assigns nodeid != -1 SqlResultSet procs; if (!atrt.doQuery("SELECT * FROM process where type <> \'mysql\'", procs)) return NDBT_FAILED; while (procs.next()) { if (procs.columnAsInt("node_id") == (unsigned)-1) { ndbout << "Found one process with node_id -1, " << "use --fix-nodeid=1 to atrt to fix this" << endl; return NDBT_FAILED; } } return NDBT_OK; }
int runRestarter(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0); NdbRestarter restarter; int i = 0; int lastId = 0; if (restarter.getNumDbNodes() < 2){ ctx->stopTest(); return NDBT_OK; } if(restarter.waitClusterStarted(60) != 0){ g_err << "Cluster failed to start" << endl; return NDBT_FAILED; } loops *= restarter.getNumDbNodes(); while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){ int id = lastId % restarter.getNumDbNodes(); int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; if(restarter.restartOneDbNode(nodeId, false, false, true) != 0){ g_err << "Failed to restartNextDbNode" << endl; result = NDBT_FAILED; break; } if(restarter.waitClusterStarted(60) != 0){ g_err << "Cluster failed to start" << endl; result = NDBT_FAILED; break; } ctx->sync_up_and_wait("PauseThreads", sync_threads); lastId++; i++; } ctx->stopTest(); return result; }
int runCheckAllNodesStarted(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; if(restarter.waitClusterStarted(1) != 0){ g_err << "All nodes was not started " << endl; return NDBT_FAILED; } return NDBT_OK; }
int runBug26457(NDBT_Context* ctx, NDBT_Step* step) { NdbRestarter res; if (res.getNumDbNodes() < 4) return NDBT_OK; int loops = ctx->getNumLoops(); while (loops --) { retry: int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); ndbout_c("master: %d next: %d", master, next); if (res.getNodeGroup(master) == res.getNodeGroup(next)) { res.restartOneDbNode(next, false, false, true); if (res.waitClusterStarted()) return NDBT_FAILED; goto retry; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 }; if (res.dumpStateOneNode(next, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(next, 7180)) return NDBT_FAILED; res.restartOneDbNode(master, false, false, true); if (res.waitClusterStarted()) return NDBT_FAILED; } return NDBT_OK; }
int runBug27003(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; static const int errnos[] = { 4025, 4026, 4027, 4028, 0 }; int node = res.getRandomNotMasterNodeId(rand()); ndbout_c("node: %d", node); if (res.restartOneDbNode(node, false, true, true)) return NDBT_FAILED; Uint32 pos = 0; for (Uint32 i = 0; i<loops; i++) { while (errnos[pos] != 0) { ndbout_c("Tesing err: %d", errnos[pos]); if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; if (res.insertErrorInNode(node, 1000)) return NDBT_FAILED; if (res.insertErrorInNode(node, errnos[pos])) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node, val2, 2)) return NDBT_FAILED; res.startNodes(&node, 1); NdbSleep_SecSleep(3); pos++; } pos = 0; } if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; res.startNodes(&node, 1); if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runDropTablesRestart(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; if (runDropTable(ctx, step) != 0) return NDBT_FAILED; if (restarter.restartAll(false) != 0) return NDBT_FAILED; if (restarter.waitClusterStarted() != 0) return NDBT_FAILED; return NDBT_OK; }
int runBug20185(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; HugoOperations hugoOps(*ctx->getTab()); Ndb* pNdb = GETNDB(step); int dump[] = { 7090, 20 } ; if (restarter.dumpStateAllNodes(dump, 2)) return NDBT_FAILED; NdbSleep_MilliSleep(3000); if(hugoOps.startTransaction(pNdb) != 0) return NDBT_FAILED; if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0) return NDBT_FAILED; if (hugoOps.execute_NoCommit(pNdb) != 0) return NDBT_FAILED; int nodeId; const int node = hugoOps.getTransaction()->getConnectedNodeId(); do { nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes()); } while (nodeId == node); if (restarter.insertErrorInAllNodes(7030)) return NDBT_FAILED; if (restarter.insertErrorInNode(nodeId, 7031)) return NDBT_FAILED; NdbSleep_MilliSleep(500); if (hugoOps.execute_Commit(pNdb) == 0) return NDBT_FAILED; NdbSleep_MilliSleep(3000); restarter.waitClusterStarted(); if (restarter.dumpStateAllNodes(dump, 1)) return NDBT_FAILED; return NDBT_OK; }
int runBug25554(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; if (restarter.getNumDbNodes() < 4) return NDBT_OK; for (int i = 0; i<loops; i++) { int master = restarter.getMasterNodeId(); int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand()); restarter.restartOneDbNode(node1, false, true, true); int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (restarter.dumpStateOneNode(master, val2, 2)) return NDBT_FAILED; if (restarter.insertErrorInNode(master, 7141)) return NDBT_FAILED; if (restarter.waitNodesNoStart(&node1, 1)) return NDBT_FAILED; if (restarter.dumpStateOneNode(node1, val2, 2)) return NDBT_FAILED; if (restarter.insertErrorInNode(node1, 932)) return NDBT_FAILED; if (restarter.startNodes(&node1, 1)) return NDBT_FAILED; int nodes[] = { master, node1 }; if (restarter.waitNodesNoStart(nodes, 2)) return NDBT_FAILED; if (restarter.startNodes(nodes, 2)) return NDBT_FAILED; if (restarter.waitClusterStarted()) return NDBT_FAILED; } return NDBT_OK; }
int runDropTablesRestart(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; Ndb* pNdb = GETNDB(step); const NdbDictionary::Table *tab = ctx->getTab(); pNdb->getDictionary()->dropTable(tab->getName()); if (restarter.restartAll(false) != 0) return NDBT_FAILED; if (restarter.waitClusterStarted() != 0) return NDBT_FAILED; return NDBT_OK; }
int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter restarter; HugoOperations hugoOps(*ctx->getTab()); Ndb* pNdb = GETNDB(step); int i = 0; while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){ g_info << i << ": "; int id = i % restarter.getNumDbNodes(); int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; restarter.insertErrorInNode(nodeId, 5041); restarter.insertErrorInAllNodes(8048 + (i & 1)); for(int j = 0; j<records; j++){ if(hugoOps.startTransaction(pNdb) != 0) return NDBT_FAILED; if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0) goto err; int res; if((res = hugoOps.execute_Commit(pNdb)) == 4119) goto done; if(res != 0) goto err; if(hugoOps.closeTransaction(pNdb) != 0) return NDBT_FAILED; } done: if(hugoOps.closeTransaction(pNdb) != 0) return NDBT_FAILED; i++; restarter.waitClusterStarted(60) ; } return result; err: hugoOps.closeTransaction(pNdb); return NDBT_FAILED; }
static int restart() { g_info << "Restarting cluster" << endl; g_hugo_ops->closeTransaction(g_ndb); disconnect_ndb(); delete g_hugo_ops; require(!g_restarter.restartAll()); require(!g_restarter.waitClusterStarted(30)); require(!connect_ndb()); g_table = g_ndb->getDictionary()->getTable(g_tablename); require(g_table); require(g_hugo_ops = new HugoOperations(* g_table)); require(!g_hugo_ops->startTransaction(g_ndb)); return 0; }
int runBug32160(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); NdbRestarter res; if (res.getNumDbNodes() < 2) { return NDBT_OK; } int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); if (res.insertErrorInNode(next, 7194)) { return NDBT_FAILED; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(master, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(master, 7193)) return NDBT_FAILED; int val3[] = { 7099 }; if (res.dumpStateOneNode(master, val3, 1)) return NDBT_FAILED; if (res.waitNodesNoStart(&master, 1)) return NDBT_FAILED; if (res.startNodes(&master, 1)) return NDBT_FAILED; if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug27283(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; if (res.getNumDbNodes() < 2) { return NDBT_OK; } static const int errnos[] = { 7181, 7182, 0 }; Uint32 pos = 0; for (Uint32 i = 0; i<loops; i++) { while (errnos[pos] != 0) { int master = res.getMasterNodeId(); int next = res.getNextMasterNodeId(master); int next2 = res.getNextMasterNodeId(next); int node = (i & 1) ? next : next2; ndbout_c("Tesing err: %d", errnos[pos]); if (res.insertErrorInNode(next, errnos[pos])) return NDBT_FAILED; NdbSleep_SecSleep(3); if (res.waitClusterStarted()) return NDBT_FAILED; pos++; } pos = 0; } return NDBT_OK; }
int runBug26481(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); NdbRestarter res; int node = res.getRandomNotMasterNodeId(rand()); ndbout_c("node: %d", node); if (res.restartOneDbNode(node, true, true, true)) return NDBT_FAILED; if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(node, 7018)) return NDBT_FAILED; if (res.startNodes(&node, 1)) return NDBT_FAILED; res.waitNodesStartPhase(&node, 1, 3); if (res.waitNodesNoStart(&node, 1)) return NDBT_FAILED; res.startNodes(&node, 1); if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug15685(NDBT_Context* ctx, NDBT_Step* step){ Ndb* pNdb = GETNDB(step); HugoOperations hugoOps(*ctx->getTab()); NdbRestarter restarter; HugoTransactions hugoTrans(*ctx->getTab()); if (hugoTrans.loadTable(GETNDB(step), 10) != 0){ return NDBT_FAILED; } if(hugoOps.startTransaction(pNdb) != 0) goto err; if(hugoOps.pkUpdateRecord(pNdb, 0, 1, rand()) != 0) goto err; if(hugoOps.execute_NoCommit(pNdb) != 0) goto err; if (restarter.insertErrorInAllNodes(5100)) return NDBT_FAILED; hugoOps.execute_Rollback(pNdb); if (restarter.waitClusterStarted() != 0) goto err; if (restarter.insertErrorInAllNodes(0)) return NDBT_FAILED; ctx->stopTest(); return NDBT_OK; err: ctx->stopTest(); return NDBT_FAILED; }
int runBackupUndoWaitStarted(NDBT_Context* ctx, NDBT_Step* step){ NdbBackup backup(GETNDB(step)->getNodeId()+1); unsigned backupId = 0; int undoError = 10041; NdbRestarter restarter; if(restarter.waitClusterStarted(60)){ g_err << "waitClusterStarted failed"<< endl; return NDBT_FAILED; } if (restarter.insertErrorInAllNodes(undoError) != 0) { g_err << "Error insert failed" << endl; return NDBT_FAILED; } // start backup wait started if (backup.start(backupId, 1, 0, 1) == -1){ return NDBT_FAILED; } ndbout << "Started backup " << backupId << endl; ctx->setProperty("BackupId", backupId); return NDBT_OK; }
int runRestarter(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0); int sleep0 = ctx->getProperty("Sleep0", (unsigned)0); int sleep1 = ctx->getProperty("Sleep1", (unsigned)0); int randnode = ctx->getProperty("RandNode", (unsigned)0); NdbRestarter restarter; int i = 0; int lastId = 0; if (restarter.getNumDbNodes() < 2) { ctx->stopTest(); return NDBT_OK; } if(restarter.waitClusterStarted() != 0) { g_err << "Cluster failed to start" << endl; return NDBT_FAILED; } if (loops > restarter.getNumDbNodes()) loops = restarter.getNumDbNodes(); while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()) { int id = lastId % restarter.getNumDbNodes(); if (randnode == 1) { id = rand() % restarter.getNumDbNodes(); } int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; if(restarter.restartOneDbNode(nodeId, false, true, true) != 0) { g_err << "Failed to restartNextDbNode" << endl; result = NDBT_FAILED; break; } if (restarter.waitNodesNoStart(&nodeId, 1)) { g_err << "Failed to waitNodesNoStart" << endl; result = NDBT_FAILED; break; } if (sleep1) NdbSleep_MilliSleep(sleep1); if (restarter.startNodes(&nodeId, 1)) { g_err << "Failed to start node" << endl; result = NDBT_FAILED; break; } if(restarter.waitClusterStarted() != 0) { g_err << "Cluster failed to start" << endl; result = NDBT_FAILED; break; } if (sleep0) NdbSleep_MilliSleep(sleep0); ctx->sync_up_and_wait("PauseThreads", sync_threads); lastId++; i++; } ctx->stopTest(); return result; }
int NdbBackup::NF(NdbRestarter& _restarter, int *NFDuringBackup_codes, const int sz, bool onMaster){ int nNodes = _restarter.getNumDbNodes(); { if(nNodes == 1) return NDBT_OK; int nodeId = _restarter.getMasterNodeId(); CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0, "Could not restart node "<< nodeId); CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = NFDuringBackup_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::NF node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0, "failed to set RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; NdbSleep_SecSleep(1); g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0, "waitNodesNoStart failed"); g_info << "number of nodes running " << _restarter.getNumDbNodes() << endl; if (_restarter.getNumDbNodes() != nNodes) { g_err << "Failure: cluster not up" << endl; return NDBT_FAILED; } g_info << "starting new backup" << endl; CHECK(start(backupId) == 0, "failed to start backup"); g_info << "(which should succeed) started with id = " << backupId << endl; g_info << "starting node" << endl; CHECK(_restarter.startNodes(&nodeId, 1) == 0, "failed to start node"); CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); g_info << "node started" << endl; int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(1); } return NDBT_OK; }
int runRestoreBankAndVerify(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; NdbBackup backup(GETNDB(step)->getNodeId()+1); unsigned minBackupId = ctx->getProperty("MinBackupId"); unsigned maxBackupId = ctx->getProperty("MaxBackupId"); unsigned backupId = minBackupId; int result = NDBT_OK; int errSumAccounts = 0; int errValidateGL = 0; ndbout << " maxBackupId = " << maxBackupId << endl; ndbout << " minBackupId = " << minBackupId << endl; while (backupId <= maxBackupId){ // TEMPORARY FIX // To erase all tables from cache(s) // To be removed, maybe replaced by ndb.invalidate(); runDropTable(ctx,step); { Bank bank(ctx->m_cluster_connection); if (bank.dropBank() != NDBT_OK){ result = NDBT_FAILED; break; } } // END TEMPORARY FIX ndbout << "Performing restart" << endl; if (restarter.restartAll(false) != 0) return NDBT_FAILED; if (restarter.waitClusterStarted() != 0) return NDBT_FAILED; ndbout << "Restoring backup " << backupId << endl; if (backup.restore(backupId) == -1){ return NDBT_FAILED; } ndbout << "Backup " << backupId << " restored" << endl; // Let bank verify Bank bank(ctx->m_cluster_connection); int wait = 0; int yield = 1; if (bank.performSumAccounts(wait, yield) != 0){ ndbout << "bank.performSumAccounts FAILED" << endl; ndbout << " backupId = " << backupId << endl << endl; result = NDBT_FAILED; errSumAccounts++; } if (bank.performValidateAllGLs() != 0){ ndbout << "bank.performValidateAllGLs FAILED" << endl; ndbout << " backupId = " << backupId << endl << endl; result = NDBT_FAILED; errValidateGL++; } backupId++; } if (result != NDBT_OK){ ndbout << "Verification of backup failed" << endl << " errValidateGL="<<errValidateGL<<endl << " errSumAccounts="<<errSumAccounts<<endl << endl; } return result; }
int NdbBackup::Fail(NdbRestarter& _restarter, int *Fail_codes, const int sz, bool onMaster){ CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); int nNodes = _restarter.getNumDbNodes(); myRandom48Init(NdbTick_CurrentMillisecond()); for(int i = 0; i<sz; i++){ int error = Fail_codes[i]; unsigned int backupId; const int masterNodeId = _restarter.getMasterNodeId(); CHECK(masterNodeId > 0, "getMasterNodeId failed"); int nodeId; nodeId = masterNodeId; if (!onMaster) { int randomId; while (nodeId == masterNodeId) { randomId = myRandom48(nNodes); nodeId = _restarter.getDbNodeId(randomId); } } g_err << "NdbBackup::Fail node = " << nodeId << " error code = " << error << " masterNodeId = " << masterNodeId << endl; CHECK(_restarter.insertErrorInNode(nodeId, error) == 0, "failed to set error insert"); g_info << "error inserted" << endl; g_info << "waiting some before starting backup" << endl; g_info << "starting backup" << endl; int r = start(backupId); g_info << "r = " << r << " (which should fail) started with id = " << backupId << endl; if (r == 0) { g_err << "Backup should have failed on error_insertion " << error << endl << "Master = " << masterNodeId << "Node = " << nodeId << endl; return NDBT_FAILED; } CHECK(_restarter.waitClusterStarted() == 0, "waitClusterStarted failed"); CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0, "failed to set error insert"); NdbSleep_SecSleep(5); int val2[] = { 24, 2424 }; CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0, "failed to check backup resources RestartOnErrorInsert"); } return NDBT_OK; }
int runTestSingleUserMode(NDBT_Context* ctx, NDBT_Step* step){ int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); NdbRestarter restarter; char tabName[255]; strncpy(tabName, ctx->getTab()->getName(), 255); ndbout << "tabName="<<tabName<<endl; int i = 0; int count; HugoTransactions hugoTrans(*ctx->getTab()); UtilTransactions utilTrans(*ctx->getTab()); while (i<loops && result == NDBT_OK) { g_info << i << ": "; int timeout = 120; // Test that the single user mode api can do everything CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0); CHECK(restarter.waitClusterSingleUser(timeout) == 0); CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0); CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0); CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0); CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0); CHECK(count == records); CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0); CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0); CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0); CHECK(count == (records/2)); CHECK(utilTrans.clearTable(pNdb, records/2) == 0); CHECK(restarter.exitSingleUserMode() == 0); CHECK(restarter.waitClusterStarted(timeout) == 0); // Test create index in single user mode CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0); CHECK(restarter.waitClusterSingleUser(timeout) == 0); CHECK(create_index_on_pk(pNdb, tabName) == 0); CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0); CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0); CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0); CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0); CHECK(count == records); CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0); CHECK(drop_index_on_pk(pNdb, tabName) == 0); CHECK(restarter.exitSingleUserMode() == 0); CHECK(restarter.waitClusterStarted(timeout) == 0); // Test recreate index in single user mode CHECK(create_index_on_pk(pNdb, tabName) == 0); CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0); CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0); CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0); CHECK(restarter.waitClusterSingleUser(timeout) == 0); CHECK(drop_index_on_pk(pNdb, tabName) == 0); CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0); CHECK(create_index_on_pk(pNdb, tabName) == 0); CHECK(restarter.exitSingleUserMode() == 0); CHECK(restarter.waitClusterStarted(timeout) == 0); CHECK(drop_index_on_pk(pNdb, tabName) == 0); CHECK(utilTrans.clearTable(GETNDB(step), records) == 0); ndbout << "Restarting cluster" << endl; CHECK(restarter.restartAll() == 0); CHECK(restarter.waitClusterStarted(timeout) == 0); CHECK(pNdb->waitUntilReady(timeout) == 0); i++; } return result; }
int runRestarter(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int waitprogress = ctx->getProperty("WaitProgress", (unsigned)0); int randnode = ctx->getProperty("RandNode", (unsigned)0); NdbRestarter restarter; int i = 0; int lastId = 0; if (restarter.getNumDbNodes() < 2){ ctx->stopTest(); return NDBT_OK; } if(restarter.waitClusterStarted() != 0){ g_err << "Cluster failed to start" << endl; return NDBT_FAILED; } loops *= (restarter.getNumDbNodes() > 2 ? 2 : restarter.getNumDbNodes()); if (loops < restarter.getNumDbNodes()) loops = restarter.getNumDbNodes(); NdbSleep_MilliSleep(200); Uint32 running = ctx->getProperty("Running", (Uint32)0); while (running == 0 && !ctx->isTestStopped()) { NdbSleep_MilliSleep(100); running = ctx->getProperty("Running", (Uint32)0); } if (ctx->isTestStopped()) return NDBT_FAILED; while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){ int id = lastId % restarter.getNumDbNodes(); if (randnode == 1) { id = rand() % restarter.getNumDbNodes(); } int nodeId = restarter.getDbNodeId(id); ndbout << "Restart node " << nodeId << endl; if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){ g_err << "Failed to restartNextDbNode" << endl; result = NDBT_FAILED; break; } if (restarter.waitNodesNoStart(&nodeId, 1)) { g_err << "Failed to waitNodesNoStart" << endl; result = NDBT_FAILED; break; } if (waitprogress) { Uint32 maxwait = 30; ndbout_c("running: 0x%.8x", running); for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++) { ctx->setProperty("Running", (Uint32)0); for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--) { if ((ctx->getProperty("Running", (Uint32)0) & running) == running) goto ok; NdbSleep_SecSleep(1); } if (ctx->isTestStopped()) { g_err << "Test stopped while waiting for progress!" << endl; return NDBT_FAILED; } g_err << "No progress made!!" << endl; return NDBT_FAILED; ok: g_err << "Progress made!! " << endl; } } if (restarter.startNodes(&nodeId, 1)) { g_err << "Failed to start node" << endl; result = NDBT_FAILED; break; } if(restarter.waitClusterStarted() != 0){ g_err << "Cluster failed to start" << endl; result = NDBT_FAILED; break; } if (waitprogress) { Uint32 maxwait = 30; ndbout_c("running: 0x%.8x", running); for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++) { ctx->setProperty("Running", (Uint32)0); for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--) { if ((ctx->getProperty("Running", (Uint32)0) & running) == running) goto ok2; NdbSleep_SecSleep(1); } if (ctx->isTestStopped()) { g_err << "Test stopped while waiting for progress!" << endl; return NDBT_FAILED; } g_err << "No progress made!!" << endl; return NDBT_FAILED; ok2: g_err << "Progress made!! " << endl; ctx->setProperty("Running", (Uint32)0); } } lastId++; i++; } ctx->stopTest(); return result; }
int runBug28717(NDBT_Context* ctx, NDBT_Step* step) { int result = NDBT_OK; int loops = ctx->getNumLoops(); int records = ctx->getNumRecords(); Ndb* pNdb = GETNDB(step); NdbRestarter res; if (res.getNumDbNodes() < 4) { return NDBT_OK; } int master = res.getMasterNodeId(); int node0 = res.getRandomNodeOtherNodeGroup(master, rand()); int node1 = res.getRandomNodeSameNodeGroup(node0, rand()); ndbout_c("master: %d node0: %d node1: %d", master, node0, node1); if (res.restartOneDbNode(node0, false, true, true)) { return NDBT_FAILED; } { int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 }; NdbLogEventHandle handle = ndb_mgm_create_logevent_handle(res.handle, filter); int dump[] = { DumpStateOrd::DihStartLcpImmediately }; struct ndb_logevent event; for (Uint32 i = 0; i<3; i++) { res.dumpStateOneNode(master, dump, 1); while(ndb_logevent_get_next(handle, &event, 0) >= 0 && event.type != NDB_LE_LocalCheckpointStarted); while(ndb_logevent_get_next(handle, &event, 0) >= 0 && event.type != NDB_LE_LocalCheckpointCompleted); } } if (res.waitNodesNoStart(&node0, 1)) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (res.dumpStateOneNode(node0, val2, 2)) return NDBT_FAILED; if (res.insertErrorInNode(node0, 5010)) return NDBT_FAILED; if (res.insertErrorInNode(node1, 1001)) return NDBT_FAILED; if (res.startNodes(&node0, 1)) return NDBT_FAILED; NdbSleep_SecSleep(3); if (res.insertErrorInNode(node1, 0)) return NDBT_FAILED; if (res.waitNodesNoStart(&node0, 1)) return NDBT_FAILED; if (res.startNodes(&node0, 1)) return NDBT_FAILED; if (res.waitClusterStarted()) return NDBT_FAILED; return NDBT_OK; }
int runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){ // Assume two replicas NdbRestarter restarter; if (restarter.getNumDbNodes() < 2) { ctx->stopTest(); return NDBT_OK; } Uint32 cnt = restarter.getNumDbNodes(); for(int loop = 0; loop < ctx->getNumLoops(); loop++) { int partition0[256]; int partition1[256]; bzero(partition0, sizeof(partition0)); bzero(partition1, sizeof(partition1)); Bitmask<4> nodesmask; Uint32 node1 = restarter.getDbNodeId(rand()%cnt); for (Uint32 i = 0; i<cnt/2; i++) { do { int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand()); if (tmp == -1) break; node1 = tmp; } while(nodesmask.get(node1)); partition0[i] = node1; partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand()); ndbout_c("nodes %d %d", node1, partition1[i]); assert(!nodesmask.get(node1)); assert(!nodesmask.get(partition1[i])); nodesmask.set(node1); nodesmask.set(partition1[i]); } ndbout_c("done"); if (restarter.restartAll(false, true, false)) return NDBT_FAILED; int dump[255]; dump[0] = 9000; memcpy(dump + 1, partition0, sizeof(int)*cnt/2); for (Uint32 i = 0; i<cnt/2; i++) if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2)) return NDBT_FAILED; dump[0] = 9000; memcpy(dump + 1, partition1, sizeof(int)*cnt/2); for (Uint32 i = 0; i<cnt/2; i++) if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2)) return NDBT_FAILED; int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (restarter.dumpStateAllNodes(val2, 2)) return NDBT_FAILED; if (restarter.insertErrorInAllNodes(932)) return NDBT_FAILED; if (restarter.startAll()) return NDBT_FAILED; if (restarter.waitClusterStartPhase(2)) return NDBT_FAILED; dump[0] = 9001; for (Uint32 i = 0; i<cnt/2; i++) if (restarter.dumpStateAllNodes(dump, 2)) return NDBT_FAILED; if (restarter.waitClusterNoStart(30)) if (restarter.waitNodesNoStart(partition0, cnt/2, 10)) if (restarter.waitNodesNoStart(partition1, cnt/2, 10)) return NDBT_FAILED; if (restarter.startAll()) return NDBT_FAILED; if (restarter.waitClusterStarted()) return NDBT_FAILED; } return NDBT_OK; }
int runBug18414(NDBT_Context* ctx, NDBT_Step* step){ NdbRestarter restarter; if (restarter.getNumDbNodes() < 2) { ctx->stopTest(); return NDBT_OK; } Ndb* pNdb = GETNDB(step); HugoOperations hugoOps(*ctx->getTab()); HugoTransactions hugoTrans(*ctx->getTab()); int loop = 0; do { if(hugoOps.startTransaction(pNdb) != 0) goto err; if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0) goto err; if(hugoOps.execute_NoCommit(pNdb) != 0) goto err; int node1 = hugoOps.getTransaction()->getConnectedNodeId(); int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand()); if (node1 == -1 || node2 == -1) break; if (loop & 1) { if (restarter.insertErrorInNode(node1, 8050)) goto err; } int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; if (restarter.dumpStateOneNode(node2, val2, 2)) goto err; if (restarter.insertErrorInNode(node2, 5003)) goto err; int res= hugoOps.execute_Rollback(pNdb); if (restarter.waitNodesNoStart(&node2, 1) != 0) goto err; if (restarter.insertErrorInAllNodes(0)) goto err; if (restarter.startNodes(&node2, 1) != 0) goto err; if (restarter.waitClusterStarted() != 0) goto err; if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0) goto err; hugoOps.closeTransaction(pNdb); } while(++loop < 5); return NDBT_OK; err: hugoOps.closeTransaction(pNdb); return NDBT_FAILED; }
int runPostUpgradeChecks(NDBT_Context* ctx, NDBT_Step* step) { /** * Table will be dropped/recreated * automatically by NDBT... * so when we enter here, this is already tested */ NdbBackup backup; ndbout << "Starting backup..." << flush; if (backup.start() != 0) { ndbout << "Failed" << endl; return NDBT_FAILED; } ndbout << "done" << endl; if ((ctx->getProperty("NoDDL", Uint32(0)) == 0) && (ctx->getProperty("KeepFS", Uint32(0)) != 0)) { /** * Bug48227 * Upgrade with FS 6.3->7.0, followed by table * create, followed by Sys restart resulted in * table loss. */ Ndb* pNdb = GETNDB(step); NdbDictionary::Dictionary *pDict = pNdb->getDictionary(); { NdbDictionary::Dictionary::List l; pDict->listObjects(l); for (Uint32 i = 0; i<l.count; i++) ndbout_c("found %u : %s", l.elements[i].id, l.elements[i].name); } pDict->dropTable("I3"); if (NDBT_Tables::createTable(pNdb, "I3")) { ndbout_c("Failed to create table!"); ndbout << pDict->getNdbError() << endl; return NDBT_FAILED; } { NdbDictionary::Dictionary::List l; pDict->listObjects(l); for (Uint32 i = 0; i<l.count; i++) ndbout_c("found %u : %s", l.elements[i].id, l.elements[i].name); } NdbRestarter res; if (res.restartAll() != 0) { ndbout_c("restartAll() failed"); return NDBT_FAILED; } if (res.waitClusterStarted() != 0) { ndbout_c("waitClusterStarted() failed"); return NDBT_FAILED; } if (pDict->getTable("I3") == 0) { ndbout_c("Table disappered"); return NDBT_FAILED; } } return NDBT_OK; }