int
runBug27003(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter res;
static const int errnos[] = { 4025, 4026, 4027, 4028, 0 };
int node = res.getRandomNotMasterNodeId(rand());
ndbout_c("node: %d", node);
if (res.restartOneDbNode(node, false, true, true))
return NDBT_FAILED;
Uint32 pos = 0;
for (Uint32 i = 0; i<loops; i++)
{
while (errnos[pos] != 0)
{
ndbout_c("Tesing err: %d", errnos[pos]);
if (res.waitNodesNoStart(&node, 1))
return NDBT_FAILED;
if (res.insertErrorInNode(node, 1000))
return NDBT_FAILED;
if (res.insertErrorInNode(node, errnos[pos]))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (res.dumpStateOneNode(node, val2, 2))
return NDBT_FAILED;
res.startNodes(&node, 1);
NdbSleep_SecSleep(3);
pos++;
}
pos = 0;
}
if (res.waitNodesNoStart(&node, 1))
return NDBT_FAILED;
res.startNodes(&node, 1);
if (res.waitClusterStarted())
return NDBT_FAILED;
return NDBT_OK;
}
int runBug15587(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
Uint32 tableId = ctx->getTab()->getTableId();
int dump[2] = { DumpStateOrd::LqhErrorInsert5042, 0 };
dump[1] = tableId;
int nodeId = restarter.getDbNodeId(1);
ndbout << "Restart node " << nodeId << endl;
if (restarter.restartOneDbNode(nodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&nodeId, 1))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateOneNode(nodeId, val2, 2))
return NDBT_FAILED;
if (restarter.dumpStateOneNode(nodeId, dump, 2))
return NDBT_FAILED;
if (restarter.startNodes(&nodeId, 1))
return NDBT_FAILED;
restarter.waitNodesStartPhase(&nodeId, 1, 3);
if (restarter.waitNodesNoStart(&nodeId, 1))
return NDBT_FAILED;
if (restarter.dumpStateOneNode(nodeId, val2, 1))
return NDBT_FAILED;
if (restarter.startNodes(&nodeId, 1))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&nodeId, 1))
return NDBT_FAILED;
ctx->stopTest();
return NDBT_OK;
}
int runBug15632(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
int nodeId = restarter.getDbNodeId(1);
ndbout << "Restart node " << nodeId << endl;
if (restarter.restartOneDbNode(nodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&nodeId, 1))
return NDBT_FAILED;
if (restarter.insertErrorInNode(nodeId, 7165))
return NDBT_FAILED;
if (restarter.startNodes(&nodeId, 1))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&nodeId, 1))
return NDBT_FAILED;
if (restarter.restartOneDbNode(nodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&nodeId, 1))
return NDBT_FAILED;
if (restarter.insertErrorInNode(nodeId, 7171))
return NDBT_FAILED;
if (restarter.startNodes(&nodeId, 1))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&nodeId, 1))
return NDBT_FAILED;
ctx->stopTest();
return NDBT_OK;
}
int runBug25554(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 4)
return NDBT_OK;
for (int i = 0; i<loops; i++)
{
int master = restarter.getMasterNodeId();
int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
restarter.restartOneDbNode(node1, false, true, true);
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateOneNode(master, val2, 2))
return NDBT_FAILED;
if (restarter.insertErrorInNode(master, 7141))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&node1, 1))
return NDBT_FAILED;
if (restarter.dumpStateOneNode(node1, val2, 2))
return NDBT_FAILED;
if (restarter.insertErrorInNode(node1, 932))
return NDBT_FAILED;
if (restarter.startNodes(&node1, 1))
return NDBT_FAILED;
int nodes[] = { master, node1 };
if (restarter.waitNodesNoStart(nodes, 2))
return NDBT_FAILED;
if (restarter.startNodes(nodes, 2))
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
}
return NDBT_OK;
}
int
runBug29364(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
Ndb* pNdb = GETNDB(step);
HugoTransactions hugoTrans(*ctx->getTab());
if (restarter.getNumDbNodes() < 4)
return NDBT_OK;
int dump0[] = { 9000, 0 } ;
int dump1[] = { 9001, 0 } ;
Uint32 ownNode = refToNode(pNdb->getReference());
dump0[1] = ownNode;
for (; loops; loops --)
{
int node0 = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
int node1 = restarter.getRandomNodeOtherNodeGroup(node0, rand());
restarter.restartOneDbNode(node0, false, true, true);
restarter.waitNodesNoStart(&node0, 1);
restarter.startNodes(&node0, 1);
restarter.waitClusterStarted();
restarter.restartOneDbNode(node1, false, true, true);
restarter.waitNodesNoStart(&node1, 1);
if (restarter.dumpStateOneNode(node1, dump0, 2))
return NDBT_FAILED;
restarter.startNodes(&node1, 1);
do {
for (Uint32 i = 0; i < 100; i++)
{
hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
}
} while (restarter.waitClusterStarted(5) != 0);
if (restarter.dumpStateOneNode(node1, dump1, 1))
return NDBT_FAILED;
}
return NDBT_OK;
}
int
runBug26481(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter res;
int node = res.getRandomNotMasterNodeId(rand());
ndbout_c("node: %d", node);
if (res.restartOneDbNode(node, true, true, true))
return NDBT_FAILED;
if (res.waitNodesNoStart(&node, 1))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (res.dumpStateOneNode(node, val2, 2))
return NDBT_FAILED;
if (res.insertErrorInNode(node, 7018))
return NDBT_FAILED;
if (res.startNodes(&node, 1))
return NDBT_FAILED;
res.waitNodesStartPhase(&node, 1, 3);
if (res.waitNodesNoStart(&node, 1))
return NDBT_FAILED;
res.startNodes(&node, 1);
if (res.waitClusterStarted())
return NDBT_FAILED;
return NDBT_OK;
}
int
runBug16772(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
int aliveNodeId = restarter.getRandomNotMasterNodeId(rand());
int deadNodeId = aliveNodeId;
while (deadNodeId == aliveNodeId)
deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
if (restarter.insertErrorInNode(aliveNodeId, 930))
return NDBT_FAILED;
if (restarter.restartOneDbNode(deadNodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&deadNodeId, 1))
return NDBT_FAILED;
if (restarter.startNodes(&deadNodeId, 1))
return NDBT_FAILED;
// It should now be hanging since we throw away NDB_FAILCONF
int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10);
// So this should fail...i.e it should not reach startphase 3
// Now send a NDB_FAILCONF for deadNo
int dump[] = { 7020, 323, 252, 0 };
dump[3] = deadNodeId;
if (restarter.dumpStateOneNode(aliveNodeId, dump, 4))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&deadNodeId, 1))
return NDBT_FAILED;
return ret ? NDBT_OK : NDBT_FAILED;
}
int
runBug32160(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
Ndb* pNdb = GETNDB(step);
NdbRestarter res;
if (res.getNumDbNodes() < 2)
{
return NDBT_OK;
}
int master = res.getMasterNodeId();
int next = res.getNextMasterNodeId(master);
if (res.insertErrorInNode(next, 7194))
{
return NDBT_FAILED;
}
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (res.dumpStateOneNode(master, val2, 2))
return NDBT_FAILED;
if (res.insertErrorInNode(master, 7193))
return NDBT_FAILED;
int val3[] = { 7099 };
if (res.dumpStateOneNode(master, val3, 1))
return NDBT_FAILED;
if (res.waitNodesNoStart(&master, 1))
return NDBT_FAILED;
if (res.startNodes(&master, 1))
return NDBT_FAILED;
if (res.waitClusterStarted())
return NDBT_FAILED;
return NDBT_OK;
}
int runBug25364(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
NdbRestarter restarter;
Ndb* pNdb = GETNDB(step);
int loops = ctx->getNumLoops();
if (restarter.getNumDbNodes() < 4)
return NDBT_OK;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
for (; loops; loops --)
{
int master = restarter.getMasterNodeId();
int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
int second = restarter.getRandomNodeSameNodeGroup(victim, rand());
int dump[] = { 935, victim } ;
if (restarter.dumpStateOneNode(master, dump, 2))
return NDBT_FAILED;
if (restarter.dumpStateOneNode(master, val2, 2))
return NDBT_FAILED;
if (restarter.restartOneDbNode(second, false, true, true))
return NDBT_FAILED;
int nodes[2] = { master, second };
if (restarter.waitNodesNoStart(nodes, 2))
return NDBT_FAILED;
restarter.startNodes(nodes, 2);
if (restarter.waitNodesStarted(nodes, 2))
return NDBT_FAILED;
}
return NDBT_OK;
}
int
runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){
// Assume two replicas
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
Uint32 cnt = restarter.getNumDbNodes();
for(int loop = 0; loop < ctx->getNumLoops(); loop++)
{
int partition0[256];
int partition1[256];
bzero(partition0, sizeof(partition0));
bzero(partition1, sizeof(partition1));
Bitmask<4> nodesmask;
Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
for (Uint32 i = 0; i<cnt/2; i++)
{
do {
int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
if (tmp == -1)
break;
node1 = tmp;
} while(nodesmask.get(node1));
partition0[i] = node1;
partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
ndbout_c("nodes %d %d", node1, partition1[i]);
assert(!nodesmask.get(node1));
assert(!nodesmask.get(partition1[i]));
nodesmask.set(node1);
nodesmask.set(partition1[i]);
}
ndbout_c("done");
if (restarter.restartAll(false, true, false))
return NDBT_FAILED;
int dump[255];
dump[0] = 9000;
memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
return NDBT_FAILED;
dump[0] = 9000;
memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateAllNodes(val2, 2))
return NDBT_FAILED;
if (restarter.insertErrorInAllNodes(932))
return NDBT_FAILED;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStartPhase(2))
return NDBT_FAILED;
dump[0] = 9001;
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateAllNodes(dump, 2))
return NDBT_FAILED;
if (restarter.waitClusterNoStart(30))
if (restarter.waitNodesNoStart(partition0, cnt/2, 10))
if (restarter.waitNodesNoStart(partition1, cnt/2, 10))
return NDBT_FAILED;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
}
return NDBT_OK;
}
int
runBug18414(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
Ndb* pNdb = GETNDB(step);
HugoOperations hugoOps(*ctx->getTab());
HugoTransactions hugoTrans(*ctx->getTab());
int loop = 0;
do
{
if(hugoOps.startTransaction(pNdb) != 0)
goto err;
if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0)
goto err;
if(hugoOps.execute_NoCommit(pNdb) != 0)
goto err;
int node1 = hugoOps.getTransaction()->getConnectedNodeId();
int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
if (node1 == -1 || node2 == -1)
break;
if (loop & 1)
{
if (restarter.insertErrorInNode(node1, 8050))
goto err;
}
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateOneNode(node2, val2, 2))
goto err;
if (restarter.insertErrorInNode(node2, 5003))
goto err;
int res= hugoOps.execute_Rollback(pNdb);
if (restarter.waitNodesNoStart(&node2, 1) != 0)
goto err;
if (restarter.insertErrorInAllNodes(0))
goto err;
if (restarter.startNodes(&node2, 1) != 0)
goto err;
if (restarter.waitClusterStarted() != 0)
goto err;
if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0)
goto err;
hugoOps.closeTransaction(pNdb);
} while(++loop < 5);
return NDBT_OK;
err:
hugoOps.closeTransaction(pNdb);
return NDBT_FAILED;
}
int runLateCommit(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
int i = 0;
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
g_info << i << ": ";
if(hugoOps.startTransaction(pNdb) != 0)
return NDBT_FAILED;
if(hugoOps.pkUpdateRecord(pNdb, 1, 128) != 0)
return NDBT_FAILED;
if(hugoOps.execute_NoCommit(pNdb) != 0)
return NDBT_FAILED;
Uint32 transNode= hugoOps.getTransaction()->getConnectedNodeId();
int id = i % restarter.getNumDbNodes();
int nodeId;
while((nodeId = restarter.getDbNodeId(id)) == transNode)
id = (id + 1) % restarter.getNumDbNodes();
ndbout << "Restart node " << nodeId << endl;
restarter.restartOneDbNode(nodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true);
restarter.waitNodesNoStart(&nodeId, 1);
int res;
if(i & 1)
res= hugoOps.execute_Commit(pNdb);
else
res= hugoOps.execute_Rollback(pNdb);
ndbout_c("res= %d", res);
hugoOps.closeTransaction(pNdb);
restarter.startNodes(&nodeId, 1);
restarter.waitNodesStarted(&nodeId, 1);
if(i & 1)
{
if(res != 286)
return NDBT_FAILED;
}
else
{
if(res != 0)
return NDBT_FAILED;
}
i++;
}
return NDBT_OK;
}
int
NdbBackup::NF(NdbRestarter& _restarter, int *NFDuringBackup_codes, const int sz, bool onMaster){
int nNodes = _restarter.getNumDbNodes();
{
if(nNodes == 1)
return NDBT_OK;
int nodeId = _restarter.getMasterNodeId();
CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0,
"Could not restart node "<< nodeId);
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
}
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
myRandom48Init(NdbTick_CurrentMillisecond());
for(int i = 0; i<sz; i++){
int error = NFDuringBackup_codes[i];
unsigned int backupId;
const int masterNodeId = _restarter.getMasterNodeId();
CHECK(masterNodeId > 0, "getMasterNodeId failed");
int nodeId;
nodeId = masterNodeId;
if (!onMaster) {
int randomId;
while (nodeId == masterNodeId) {
randomId = myRandom48(nNodes);
nodeId = _restarter.getDbNodeId(randomId);
}
}
g_err << "NdbBackup::NF node = " << nodeId
<< " error code = " << error << " masterNodeId = "
<< masterNodeId << endl;
int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0,
"failed to set RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(nodeId, error) == 0,
"failed to set error insert");
g_info << "error inserted" << endl;
NdbSleep_SecSleep(1);
g_info << "starting backup" << endl;
int r = start(backupId);
g_info << "r = " << r
<< " (which should fail) started with id = " << backupId << endl;
if (r == 0) {
g_err << "Backup should have failed on error_insertion " << error << endl
<< "Master = " << masterNodeId << "Node = " << nodeId << endl;
return NDBT_FAILED;
}
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
g_info << "number of nodes running " << _restarter.getNumDbNodes() << endl;
if (_restarter.getNumDbNodes() != nNodes) {
g_err << "Failure: cluster not up" << endl;
return NDBT_FAILED;
}
g_info << "starting new backup" << endl;
CHECK(start(backupId) == 0,
"failed to start backup");
g_info << "(which should succeed) started with id = " << backupId << endl;
g_info << "starting node" << endl;
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
g_info << "node started" << endl;
int val2[] = { 24, 2424 };
CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0,
"failed to check backup resources RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0,
"failed to set error insert");
NdbSleep_SecSleep(1);
}
return NDBT_OK;
}
int
runBug28717(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
Ndb* pNdb = GETNDB(step);
NdbRestarter res;
if (res.getNumDbNodes() < 4)
{
return NDBT_OK;
}
int master = res.getMasterNodeId();
int node0 = res.getRandomNodeOtherNodeGroup(master, rand());
int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
ndbout_c("master: %d node0: %d node1: %d", master, node0, node1);
if (res.restartOneDbNode(node0, false, true, true))
{
return NDBT_FAILED;
}
{
int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
NdbLogEventHandle handle =
ndb_mgm_create_logevent_handle(res.handle, filter);
int dump[] = { DumpStateOrd::DihStartLcpImmediately };
struct ndb_logevent event;
for (Uint32 i = 0; i<3; i++)
{
res.dumpStateOneNode(master, dump, 1);
while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
event.type != NDB_LE_LocalCheckpointStarted);
while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
event.type != NDB_LE_LocalCheckpointCompleted);
}
}
if (res.waitNodesNoStart(&node0, 1))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (res.dumpStateOneNode(node0, val2, 2))
return NDBT_FAILED;
if (res.insertErrorInNode(node0, 5010))
return NDBT_FAILED;
if (res.insertErrorInNode(node1, 1001))
return NDBT_FAILED;
if (res.startNodes(&node0, 1))
return NDBT_FAILED;
NdbSleep_SecSleep(3);
if (res.insertErrorInNode(node1, 0))
return NDBT_FAILED;
if (res.waitNodesNoStart(&node0, 1))
return NDBT_FAILED;
if (res.startNodes(&node0, 1))
return NDBT_FAILED;
if (res.waitClusterStarted())
return NDBT_FAILED;
return NDBT_OK;
}
int testWorker(NDBT_Context* ctx, NDBT_Step* step)
{
/* Run as a 'T1' testcase - do nothing for other tables */
if (strcmp(ctx->getTab()->getName(), "T1") != 0)
return NDBT_OK;
/* Worker step to run in a separate thread for
* blocking activities
* Generally the blocking of the DIH table definition flush
* blocks the completion of the drop table/node restarts,
* so this must be done in a separate thread to avoid
* deadlocks.
*/
while (!ctx->isTestStopped())
{
ndbout_c("Worker : waiting for request...");
ctx->getPropertyWait("DIHWritesRequest", 1);
if (!ctx->isTestStopped())
{
Uint32 req = ctx->getProperty("DIHWritesRequestType", (Uint32)0);
switch ((Tasks) req)
{
case DROP_TABLE_REQ:
{
/* Drop table */
ndbout_c("Worker : dropping table");
if (dropTable(ctx, step, 2) != NDBT_OK)
{
return NDBT_FAILED;
}
ndbout_c("Worker : table dropped.");
break;
}
case MASTER_RESTART_REQ:
{
ndbout_c("Worker : restarting Master");
NdbRestarter restarter;
int master_nodeid = restarter.getMasterNodeId();
ndbout_c("Worker : Restarting Master (%d)...", master_nodeid);
if (restarter.restartOneDbNode2(master_nodeid,
NdbRestarter::NRRF_NOSTART |
NdbRestarter::NRRF_FORCE |
NdbRestarter::NRRF_ABORT) ||
restarter.waitNodesNoStart(&master_nodeid, 1) ||
restarter.startAll())
{
ndbout_c("Worker : Error restarting Master.");
return NDBT_FAILED;
}
ndbout_c("Worker : Waiting for master to recover...");
if (restarter.waitNodesStarted(&master_nodeid, 1))
{
ndbout_c("Worker : Error waiting for Master restart");
return NDBT_FAILED;
}
ndbout_c("Worker : Master recovered.");
break;
}
case SLAVE_RESTART_REQ:
{
NdbRestarter restarter;
int slave_nodeid = restarter.getRandomNotMasterNodeId(rand());
ndbout_c("Worker : Restarting non-master (%d)...", slave_nodeid);
if (restarter.restartOneDbNode2(slave_nodeid,
NdbRestarter::NRRF_NOSTART |
NdbRestarter::NRRF_FORCE |
NdbRestarter::NRRF_ABORT) ||
restarter.waitNodesNoStart(&slave_nodeid, 1) ||
restarter.startAll())
{
ndbout_c("Worker : Error restarting Slave.");
return NDBT_FAILED;
}
ndbout_c("Worker : Waiting for slave to recover...");
if (restarter.waitNodesStarted(&slave_nodeid, 1))
{
ndbout_c("Worker : Error waiting for Slave restart");
return NDBT_FAILED;
}
ndbout_c("Worker : Slave recovered.");
break;
}
default:
{
break;
}
}
}
ctx->setProperty("DIHWritesRequestType", (Uint32) 0);
ctx->setProperty("DIHWritesRequest", (Uint32) 2);
}
ndbout_c("Worker, done.");
return NDBT_OK;
}
int
runRestarter(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int waitprogress = ctx->getProperty("WaitProgress", (unsigned)0);
int randnode = ctx->getProperty("RandNode", (unsigned)0);
NdbRestarter restarter;
int i = 0;
int lastId = 0;
if (restarter.getNumDbNodes() < 2){
ctx->stopTest();
return NDBT_OK;
}
if(restarter.waitClusterStarted() != 0){
g_err << "Cluster failed to start" << endl;
return NDBT_FAILED;
}
loops *= (restarter.getNumDbNodes() > 2 ? 2 : restarter.getNumDbNodes());
if (loops < restarter.getNumDbNodes())
loops = restarter.getNumDbNodes();
NdbSleep_MilliSleep(200);
Uint32 running = ctx->getProperty("Running", (Uint32)0);
while (running == 0 && !ctx->isTestStopped())
{
NdbSleep_MilliSleep(100);
running = ctx->getProperty("Running", (Uint32)0);
}
if (ctx->isTestStopped())
return NDBT_FAILED;
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
int id = lastId % restarter.getNumDbNodes();
if (randnode == 1)
{
id = rand() % restarter.getNumDbNodes();
}
int nodeId = restarter.getDbNodeId(id);
ndbout << "Restart node " << nodeId << endl;
if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){
g_err << "Failed to restartNextDbNode" << endl;
result = NDBT_FAILED;
break;
}
if (restarter.waitNodesNoStart(&nodeId, 1))
{
g_err << "Failed to waitNodesNoStart" << endl;
result = NDBT_FAILED;
break;
}
if (waitprogress)
{
Uint32 maxwait = 30;
ndbout_c("running: 0x%.8x", running);
for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++)
{
ctx->setProperty("Running", (Uint32)0);
for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--)
{
if ((ctx->getProperty("Running", (Uint32)0) & running) == running)
goto ok;
NdbSleep_SecSleep(1);
}
if (ctx->isTestStopped())
{
g_err << "Test stopped while waiting for progress!" << endl;
return NDBT_FAILED;
}
g_err << "No progress made!!" << endl;
return NDBT_FAILED;
ok:
g_err << "Progress made!! " << endl;
}
}
if (restarter.startNodes(&nodeId, 1))
{
g_err << "Failed to start node" << endl;
result = NDBT_FAILED;
break;
}
if(restarter.waitClusterStarted() != 0){
g_err << "Cluster failed to start" << endl;
result = NDBT_FAILED;
break;
}
if (waitprogress)
{
Uint32 maxwait = 30;
ndbout_c("running: 0x%.8x", running);
for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++)
{
ctx->setProperty("Running", (Uint32)0);
for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--)
{
if ((ctx->getProperty("Running", (Uint32)0) & running) == running)
goto ok2;
NdbSleep_SecSleep(1);
}
if (ctx->isTestStopped())
{
g_err << "Test stopped while waiting for progress!" << endl;
return NDBT_FAILED;
}
g_err << "No progress made!!" << endl;
return NDBT_FAILED;
ok2:
g_err << "Progress made!! " << endl;
ctx->setProperty("Running", (Uint32)0);
}
}
lastId++;
i++;
}
ctx->stopTest();
return result;
}
int runRestarter(NDBT_Context* ctx, NDBT_Step* step) {
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0);
int sleep0 = ctx->getProperty("Sleep0", (unsigned)0);
int sleep1 = ctx->getProperty("Sleep1", (unsigned)0);
int randnode = ctx->getProperty("RandNode", (unsigned)0);
NdbRestarter restarter;
int i = 0;
int lastId = 0;
if (restarter.getNumDbNodes() < 2) {
ctx->stopTest();
return NDBT_OK;
}
if(restarter.waitClusterStarted() != 0) {
g_err << "Cluster failed to start" << endl;
return NDBT_FAILED;
}
if (loops > restarter.getNumDbNodes())
loops = restarter.getNumDbNodes();
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()) {
int id = lastId % restarter.getNumDbNodes();
if (randnode == 1)
{
id = rand() % restarter.getNumDbNodes();
}
int nodeId = restarter.getDbNodeId(id);
ndbout << "Restart node " << nodeId << endl;
if(restarter.restartOneDbNode(nodeId, false, true, true) != 0) {
g_err << "Failed to restartNextDbNode" << endl;
result = NDBT_FAILED;
break;
}
if (restarter.waitNodesNoStart(&nodeId, 1))
{
g_err << "Failed to waitNodesNoStart" << endl;
result = NDBT_FAILED;
break;
}
if (sleep1)
NdbSleep_MilliSleep(sleep1);
if (restarter.startNodes(&nodeId, 1))
{
g_err << "Failed to start node" << endl;
result = NDBT_FAILED;
break;
}
if(restarter.waitClusterStarted() != 0) {
g_err << "Cluster failed to start" << endl;
result = NDBT_FAILED;
break;
}
if (sleep0)
NdbSleep_MilliSleep(sleep0);
ctx->sync_up_and_wait("PauseThreads", sync_threads);
lastId++;
i++;
}
ctx->stopTest();
return result;
}
