int runFail(NDBT_Context* ctx, NDBT_Step* step){
NdbBackup backup(GETNDB(step)->getNodeId()+1);
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2){
ctx->stopTest();
return NDBT_OK;
}
if(restarter.waitClusterStarted(60) != 0){
g_err << "Cluster failed to start" << endl;
return NDBT_FAILED;
}
if (testMaster) {
if (testSlave) {
if (backup.FailMasterAsSlave(restarter) != NDBT_OK){
return NDBT_FAILED;
}
} else {
if (backup.FailMaster(restarter) != NDBT_OK){
return NDBT_FAILED;
}
}
} else {
if (backup.FailSlave(restarter) != NDBT_OK){
return NDBT_FAILED;
}
}
return NDBT_OK;
}
int runCheckStarted(NDBT_Context* ctx, NDBT_Step* step) {
// Check cluster is started
NdbRestarter restarter;
if(restarter.waitClusterStarted() != 0) {
g_err << "All nodes was not started " << endl;
return NDBT_FAILED;
}
// Check atrtclient is started
AtrtClient atrt;
if(!atrt.waitConnected()) {
g_err << "atrt server was not started " << endl;
return NDBT_FAILED;
}
// Make sure atrt assigns nodeid != -1
SqlResultSet procs;
if (!atrt.doQuery("SELECT * FROM process where type <> \'mysql\'", procs))
return NDBT_FAILED;
while (procs.next())
{
if (procs.columnAsInt("node_id") == (unsigned)-1) {
ndbout << "Found one process with node_id -1, "
<< "use --fix-nodeid=1 to atrt to fix this" << endl;
return NDBT_FAILED;
}
}
return NDBT_OK;
}
int getRandomNodeId(NdbRestarter& _restarter) {
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
return nodeId;
}
static int pause_lcp(int error)
{
int nodes = g_restarter.getNumDbNodes();
int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_INFO, 0 };
int fd = ndb_mgm_listen_event(g_restarter.handle, filter);
require(fd >= 0);
require(!g_restarter.insertErrorInAllNodes(error));
int dump[] = { DumpStateOrd::DihStartLcpImmediately };
require(!g_restarter.dumpStateAllNodes(dump, 1));
char *tmp;
char buf[1024];
SocketInputStream in(fd, 1000);
int count = 0;
do {
tmp = in.gets(buf, 1024);
if(tmp)
{
int id;
if(sscanf(tmp, "%*[^:]: LCP: %d ", &id) == 1 && id == error &&
--nodes == 0){
close(fd);
return 0;
}
}
} while(count++ < 30);
close(fd);
return -1;
}
int runRestartGciControl(NDBT_Context* ctx, NDBT_Step* step){
int records = ctx->getNumRecords();
Ndb* pNdb = GETNDB(step);
UtilTransactions utilTrans(*ctx->getTab());
NdbRestarter restarter;
// Wait until we have enough records in db
int count = 0;
while (count < records){
if (utilTrans.selectCount(pNdb, 64, &count) != 0){
ctx->stopTest();
return NDBT_FAILED;
}
}
// Restart cluster with abort
if (restarter.restartAll(false, false, true) != 0){
ctx->stopTest();
return NDBT_FAILED;
}
// Stop the other thread
ctx->stopTest();
if (restarter.waitClusterStarted(300) != 0){
return NDBT_FAILED;
}
if (pNdb->waitUntilReady() != 0){
return NDBT_FAILED;
}
return NDBT_OK;
}
int runCheckAllNodesStarted(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if(restarter.waitClusterStarted(1) != 0){
g_err << "All nodes was not started " << endl;
return NDBT_FAILED;
}
return NDBT_OK;
}
int
resetTransactionTimeout(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
int val[] = { DumpStateOrd::TcSetApplTransactionTimeout, g_org_timeout };
if(restarter.dumpStateAllNodes(val, 2) != 0){
return NDBT_FAILED;
}
return NDBT_OK;
}
int get50PercentOfNodes(NdbRestarter& restarter,
int * _nodes){
// For now simply return all nodes with even node id
// TODO Check nodegroup and return one node from each
int num50Percent = restarter.getNumDbNodes() / 2;
require(num50Percent <= MAX_NDB_NODES);
// Calculate which nodes to stop, select all even nodes
for (int i = 0; i < num50Percent; i++){
_nodes[i] = restarter.getDbNodeId(i*2);
}
return num50Percent;
}
int runDropTablesRestart(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if (runDropTable(ctx, step) != 0)
return NDBT_FAILED;
if (restarter.restartAll(false) != 0)
return NDBT_FAILED;
if (restarter.waitClusterStarted() != 0)
return NDBT_FAILED;
return NDBT_OK;
}
int
runBug57650(NDBT_Context* ctx, NDBT_Step* step)
{
NdbBackup backup(GETNDB(step)->getNodeId()+1);
NdbRestarter res;
int node0 = res.getNode(NdbRestarter::NS_RANDOM);
res.insertErrorInNode(node0, 5057);
unsigned backupId = 0;
if (backup.start(backupId) == -1)
return NDBT_FAILED;
return NDBT_OK;
}
int
resetTransactionTimeout(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
// g_org_timeout will be passed as printed int to mgm,
// then converted to Uint32 before sent to tc.
// Check convert Uint32 -> int -> Uint32 is safe
NDB_STATIC_ASSERT(UINT_MAX32 == (Uint32)(int)UINT_MAX32);
int val[] = { DumpStateOrd::TcSetApplTransactionTimeout, (int)g_org_timeout };
if(restarter.dumpStateAllNodes(val, 2) != 0){
return NDBT_FAILED;
}
return NDBT_OK;
}
static int continue_lcp(int error)
{
int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_INFO, 0 };
NDB_SOCKET_TYPE my_fd;
my_socket_invalidate(&my_fd);
#ifdef NDB_WIN
SOCKET fd;
#else
int fd;
#endif
if(error){
fd = ndb_mgm_listen_event(g_restarter.handle, filter);
#ifdef NDB_WIN
my_fd.s= fd;
#else
my_fd.fd= fd;
#endif
require(my_socket_valid(my_fd));
}
int args[] = { DumpStateOrd::LCPContinue };
if(g_restarter.dumpStateAllNodes(args, 1) != 0)
return -1;
if(error){
char *tmp;
char buf[1024];
SocketInputStream in(my_fd, 1000);
int count = 0;
int nodes = g_restarter.getNumDbNodes();
do {
tmp = in.gets(buf, 1024);
if(tmp)
{
int id;
if(sscanf(tmp, "%*[^:]: LCP: %d ", &id) == 1 && id == error &&
--nodes == 0){
my_socket_close(my_fd);
return 0;
}
}
} while(count++ < 30);
my_socket_close(my_fd);
}
return 0;
}
int runDropTablesRestart(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
Ndb* pNdb = GETNDB(step);
const NdbDictionary::Table *tab = ctx->getTab();
pNdb->getDictionary()->dropTable(tab->getName());
if (restarter.restartAll(false) != 0)
return NDBT_FAILED;
if (restarter.waitClusterStarted() != 0)
return NDBT_FAILED;
return NDBT_OK;
}
static int restart()
{
g_info << "Restarting cluster" << endl;
g_hugo_ops->closeTransaction(g_ndb);
disconnect_ndb();
delete g_hugo_ops;
require(!g_restarter.restartAll());
require(!g_restarter.waitClusterStarted(30));
require(!connect_ndb());
g_table = g_ndb->getDictionary()->getTable(g_tablename);
require(g_table);
require(g_hugo_ops = new HugoOperations(* g_table));
require(!g_hugo_ops->startTransaction(g_ndb));
return 0;
}
int runBug24717(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
Ndb* pNdb = GETNDB(step);
HugoTransactions hugoTrans(*ctx->getTab());
int dump[] = { 9002, 0 } ;
Uint32 ownNode = refToNode(pNdb->getReference());
dump[1] = ownNode;
for (; loops; loops --)
{
int nodeId = restarter.getRandomNotMasterNodeId(rand());
restarter.restartOneDbNode(nodeId, false, true, true);
restarter.waitNodesNoStart(&nodeId, 1);
if (restarter.dumpStateOneNode(nodeId, dump, 2))
return NDBT_FAILED;
restarter.startNodes(&nodeId, 1);
do {
for (Uint32 i = 0; i < 100; i++)
{
hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
}
} while (restarter.waitClusterStarted(5) != 0);
}
return NDBT_OK;
}
int
setTransactionTimeout(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
int timeout = ctx->getProperty("TransactionInactiveTimeout",TIMEOUT);
NdbConfig conf;
if (!conf.getProperty(conf.getMasterNodeId(),
NODE_TYPE_DB,
CFG_DB_TRANSACTION_INACTIVE_TIMEOUT,
&g_org_timeout)){
return NDBT_FAILED;
}
int val[] = { DumpStateOrd::TcSetApplTransactionTimeout, timeout };
if(restarter.dumpStateAllNodes(val, 2) != 0){
return NDBT_FAILED;
}
return NDBT_OK;
}
int
setDeadlockTimeout(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
int timeout = ctx->getProperty("TransactionDeadlockTimeout", TIMEOUT);
NdbConfig conf;
if (!conf.getProperty(conf.getMasterNodeId(),
NODE_TYPE_DB,
CFG_DB_TRANSACTION_DEADLOCK_TIMEOUT,
&g_org_deadlock))
return NDBT_FAILED;
g_err << "Setting timeout: " << timeout << endl;
int val[] = { DumpStateOrd::TcSetTransactionTimeout, timeout };
if(restarter.dumpStateAllNodes(val, 2) != 0){
return NDBT_FAILED;
}
return NDBT_OK;
}
int runBug20185(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
int dump[] = { 7090, 20 } ;
if (restarter.dumpStateAllNodes(dump, 2))
return NDBT_FAILED;
NdbSleep_MilliSleep(3000);
if(hugoOps.startTransaction(pNdb) != 0)
return NDBT_FAILED;
if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0)
return NDBT_FAILED;
if (hugoOps.execute_NoCommit(pNdb) != 0)
return NDBT_FAILED;
int nodeId;
const int node = hugoOps.getTransaction()->getConnectedNodeId();
do {
nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
} while (nodeId == node);
if (restarter.insertErrorInAllNodes(7030))
return NDBT_FAILED;
if (restarter.insertErrorInNode(nodeId, 7031))
return NDBT_FAILED;
NdbSleep_MilliSleep(500);
if (hugoOps.execute_Commit(pNdb) == 0)
return NDBT_FAILED;
NdbSleep_MilliSleep(3000);
restarter.waitClusterStarted();
if (restarter.dumpStateAllNodes(dump, 1))
return NDBT_FAILED;
return NDBT_OK;
}
int runBug15685(NDBT_Context* ctx, NDBT_Step* step){
Ndb* pNdb = GETNDB(step);
HugoOperations hugoOps(*ctx->getTab());
NdbRestarter restarter;
HugoTransactions hugoTrans(*ctx->getTab());
if (hugoTrans.loadTable(GETNDB(step), 10) != 0){
return NDBT_FAILED;
}
if(hugoOps.startTransaction(pNdb) != 0)
goto err;
if(hugoOps.pkUpdateRecord(pNdb, 0, 1, rand()) != 0)
goto err;
if(hugoOps.execute_NoCommit(pNdb) != 0)
goto err;
if (restarter.insertErrorInAllNodes(5100))
return NDBT_FAILED;
hugoOps.execute_Rollback(pNdb);
if (restarter.waitClusterStarted() != 0)
goto err;
if (restarter.insertErrorInAllNodes(0))
return NDBT_FAILED;
ctx->stopTest();
return NDBT_OK;
err:
ctx->stopTest();
return NDBT_FAILED;
}
int runRestarter(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0);
NdbRestarter restarter;
int i = 0;
int lastId = 0;
if (restarter.getNumDbNodes() < 2){
ctx->stopTest();
return NDBT_OK;
}
if(restarter.waitClusterStarted(60) != 0){
g_err << "Cluster failed to start" << endl;
return NDBT_FAILED;
}
loops *= restarter.getNumDbNodes();
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
int id = lastId % restarter.getNumDbNodes();
int nodeId = restarter.getDbNodeId(id);
ndbout << "Restart node " << nodeId << endl;
if(restarter.restartOneDbNode(nodeId, false, false, true) != 0){
g_err << "Failed to restartNextDbNode" << endl;
result = NDBT_FAILED;
break;
}
if(restarter.waitClusterStarted(60) != 0){
g_err << "Cluster failed to start" << endl;
result = NDBT_FAILED;
break;
}
ctx->sync_up_and_wait("PauseThreads", sync_threads);
lastId++;
i++;
}
ctx->stopTest();
return result;
}
static int continue_lcp(int error)
{
int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_INFO, 0 };
int fd = -1;
if(error){
fd = ndb_mgm_listen_event(g_restarter.handle, filter);
require(fd >= 0);
}
int args[] = { DumpStateOrd::LCPContinue };
if(g_restarter.dumpStateAllNodes(args, 1) != 0)
return -1;
if(error){
char *tmp;
char buf[1024];
SocketInputStream in(fd, 1000);
int count = 0;
int nodes = g_restarter.getNumDbNodes();
do {
tmp = in.gets(buf, 1024);
if(tmp)
{
int id;
if(sscanf(tmp, "%*[^:]: LCP: %d ", &id) == 1 && id == error &&
--nodes == 0){
close(fd);
return 0;
}
}
} while(count++ < 30);
close(fd);
}
return 0;
}
int runBackupUndoWaitStarted(NDBT_Context* ctx, NDBT_Step* step){
NdbBackup backup(GETNDB(step)->getNodeId()+1);
unsigned backupId = 0;
int undoError = 10041;
NdbRestarter restarter;
if(restarter.waitClusterStarted(60)){
g_err << "waitClusterStarted failed"<< endl;
return NDBT_FAILED;
}
if (restarter.insertErrorInAllNodes(undoError) != 0) {
g_err << "Error insert failed" << endl;
return NDBT_FAILED;
}
// start backup wait started
if (backup.start(backupId, 1, 0, 1) == -1){
return NDBT_FAILED;
}
ndbout << "Started backup " << backupId << endl;
ctx->setProperty("BackupId", backupId);
return NDBT_OK;
}
int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter restarter;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
int i = 0;
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
g_info << i << ": ";
int id = i % restarter.getNumDbNodes();
int nodeId = restarter.getDbNodeId(id);
ndbout << "Restart node " << nodeId << endl;
restarter.insertErrorInNode(nodeId, 5041);
restarter.insertErrorInAllNodes(8048 + (i & 1));
for(int j = 0; j<records; j++){
if(hugoOps.startTransaction(pNdb) != 0)
return NDBT_FAILED;
if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0)
goto err;
int res;
if((res = hugoOps.execute_Commit(pNdb)) == 4119)
goto done;
if(res != 0)
goto err;
if(hugoOps.closeTransaction(pNdb) != 0)
return NDBT_FAILED;
}
done:
if(hugoOps.closeTransaction(pNdb) != 0)
return NDBT_FAILED;
i++;
restarter.waitClusterStarted(60) ;
}
return result;
err:
hugoOps.closeTransaction(pNdb);
return NDBT_FAILED;
}
int
runBug27283(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
NdbRestarter res;
if (res.getNumDbNodes() < 2)
{
return NDBT_OK;
}
static const int errnos[] = { 7181, 7182, 0 };
Uint32 pos = 0;
for (Uint32 i = 0; i<loops; i++)
{
while (errnos[pos] != 0)
{
int master = res.getMasterNodeId();
int next = res.getNextMasterNodeId(master);
int next2 = res.getNextMasterNodeId(next);
int node = (i & 1) ? next : next2;
ndbout_c("Tesing err: %d", errnos[pos]);
if (res.insertErrorInNode(next, errnos[pos]))
return NDBT_FAILED;
NdbSleep_SecSleep(3);
if (res.waitClusterStarted())
return NDBT_FAILED;
pos++;
}
pos = 0;
}
return NDBT_OK;
}
int
runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){
// Assume two replicas
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
Uint32 cnt = restarter.getNumDbNodes();
for(int loop = 0; loop < ctx->getNumLoops(); loop++)
{
int partition0[256];
int partition1[256];
bzero(partition0, sizeof(partition0));
bzero(partition1, sizeof(partition1));
Bitmask<4> nodesmask;
Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
for (Uint32 i = 0; i<cnt/2; i++)
{
do {
int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
if (tmp == -1)
break;
node1 = tmp;
} while(nodesmask.get(node1));
partition0[i] = node1;
partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
ndbout_c("nodes %d %d", node1, partition1[i]);
assert(!nodesmask.get(node1));
assert(!nodesmask.get(partition1[i]));
nodesmask.set(node1);
nodesmask.set(partition1[i]);
}
ndbout_c("done");
if (restarter.restartAll(false, true, false))
return NDBT_FAILED;
int dump[255];
dump[0] = 9000;
memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
return NDBT_FAILED;
dump[0] = 9000;
memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
return NDBT_FAILED;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateAllNodes(val2, 2))
return NDBT_FAILED;
if (restarter.insertErrorInAllNodes(932))
return NDBT_FAILED;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStartPhase(2))
return NDBT_FAILED;
dump[0] = 9001;
for (Uint32 i = 0; i<cnt/2; i++)
if (restarter.dumpStateAllNodes(dump, 2))
return NDBT_FAILED;
if (restarter.waitClusterNoStart(30))
if (restarter.waitNodesNoStart(partition0, cnt/2, 10))
if (restarter.waitNodesNoStart(partition1, cnt/2, 10))
return NDBT_FAILED;
if (restarter.startAll())
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED;
}
return NDBT_OK;
}
int
runBug18414(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
Ndb* pNdb = GETNDB(step);
HugoOperations hugoOps(*ctx->getTab());
HugoTransactions hugoTrans(*ctx->getTab());
int loop = 0;
do
{
if(hugoOps.startTransaction(pNdb) != 0)
goto err;
if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0)
goto err;
if(hugoOps.execute_NoCommit(pNdb) != 0)
goto err;
int node1 = hugoOps.getTransaction()->getConnectedNodeId();
int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
if (node1 == -1 || node2 == -1)
break;
if (loop & 1)
{
if (restarter.insertErrorInNode(node1, 8050))
goto err;
}
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
if (restarter.dumpStateOneNode(node2, val2, 2))
goto err;
if (restarter.insertErrorInNode(node2, 5003))
goto err;
int res= hugoOps.execute_Rollback(pNdb);
if (restarter.waitNodesNoStart(&node2, 1) != 0)
goto err;
if (restarter.insertErrorInAllNodes(0))
goto err;
if (restarter.startNodes(&node2, 1) != 0)
goto err;
if (restarter.waitClusterStarted() != 0)
goto err;
if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0)
goto err;
hugoOps.closeTransaction(pNdb);
} while(++loop < 5);
return NDBT_OK;
err:
hugoOps.closeTransaction(pNdb);
return NDBT_FAILED;
}
int
runBug16772(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
if (restarter.getNumDbNodes() < 2)
{
ctx->stopTest();
return NDBT_OK;
}
int aliveNodeId = restarter.getRandomNotMasterNodeId(rand());
int deadNodeId = aliveNodeId;
while (deadNodeId == aliveNodeId)
deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
if (restarter.insertErrorInNode(aliveNodeId, 930))
return NDBT_FAILED;
if (restarter.restartOneDbNode(deadNodeId,
/** initial */ false,
/** nostart */ true,
/** abort */ true))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&deadNodeId, 1))
return NDBT_FAILED;
if (restarter.startNodes(&deadNodeId, 1))
return NDBT_FAILED;
// It should now be hanging since we throw away NDB_FAILCONF
int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10);
// So this should fail...i.e it should not reach startphase 3
// Now send a NDB_FAILCONF for deadNo
int dump[] = { 7020, 323, 252, 0 };
dump[3] = deadNodeId;
if (restarter.dumpStateOneNode(aliveNodeId, dump, 4))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&deadNodeId, 1))
return NDBT_FAILED;
return ret ? NDBT_OK : NDBT_FAILED;
}
int
runRestarter(NDBT_Context* ctx, NDBT_Step* step)
{
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int waitprogress = ctx->getProperty("WaitProgress", (unsigned)0);
int randnode = ctx->getProperty("RandNode", (unsigned)0);
NdbRestarter restarter;
int i = 0;
int lastId = 0;
if (restarter.getNumDbNodes() < 2){
ctx->stopTest();
return NDBT_OK;
}
if(restarter.waitClusterStarted() != 0){
g_err << "Cluster failed to start" << endl;
return NDBT_FAILED;
}
loops *= (restarter.getNumDbNodes() > 2 ? 2 : restarter.getNumDbNodes());
if (loops < restarter.getNumDbNodes())
loops = restarter.getNumDbNodes();
NdbSleep_MilliSleep(200);
Uint32 running = ctx->getProperty("Running", (Uint32)0);
while (running == 0 && !ctx->isTestStopped())
{
NdbSleep_MilliSleep(100);
running = ctx->getProperty("Running", (Uint32)0);
}
if (ctx->isTestStopped())
return NDBT_FAILED;
while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
int id = lastId % restarter.getNumDbNodes();
if (randnode == 1)
{
id = rand() % restarter.getNumDbNodes();
}
int nodeId = restarter.getDbNodeId(id);
ndbout << "Restart node " << nodeId << endl;
if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){
g_err << "Failed to restartNextDbNode" << endl;
result = NDBT_FAILED;
break;
}
if (restarter.waitNodesNoStart(&nodeId, 1))
{
g_err << "Failed to waitNodesNoStart" << endl;
result = NDBT_FAILED;
break;
}
if (waitprogress)
{
Uint32 maxwait = 30;
ndbout_c("running: 0x%.8x", running);
for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++)
{
ctx->setProperty("Running", (Uint32)0);
for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--)
{
if ((ctx->getProperty("Running", (Uint32)0) & running) == running)
goto ok;
NdbSleep_SecSleep(1);
}
if (ctx->isTestStopped())
{
g_err << "Test stopped while waiting for progress!" << endl;
return NDBT_FAILED;
}
g_err << "No progress made!!" << endl;
return NDBT_FAILED;
ok:
g_err << "Progress made!! " << endl;
}
}
if (restarter.startNodes(&nodeId, 1))
{
g_err << "Failed to start node" << endl;
result = NDBT_FAILED;
break;
}
if(restarter.waitClusterStarted() != 0){
g_err << "Cluster failed to start" << endl;
result = NDBT_FAILED;
break;
}
if (waitprogress)
{
Uint32 maxwait = 30;
ndbout_c("running: 0x%.8x", running);
for (Uint32 checks = 0; checks < 3 && !ctx->isTestStopped(); checks++)
{
ctx->setProperty("Running", (Uint32)0);
for (; maxwait != 0 && !ctx->isTestStopped(); maxwait--)
{
if ((ctx->getProperty("Running", (Uint32)0) & running) == running)
goto ok2;
NdbSleep_SecSleep(1);
}
if (ctx->isTestStopped())
{
g_err << "Test stopped while waiting for progress!" << endl;
return NDBT_FAILED;
}
g_err << "No progress made!!" << endl;
return NDBT_FAILED;
ok2:
g_err << "Progress made!! " << endl;
ctx->setProperty("Running", (Uint32)0);
}
}
lastId++;
i++;
}
ctx->stopTest();
return result;
}
int runTestSingleUserMode(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
Ndb* pNdb = GETNDB(step);
NdbRestarter restarter;
char tabName[255];
strncpy(tabName, ctx->getTab()->getName(), 255);
ndbout << "tabName="<<tabName<<endl;
int i = 0;
int count;
HugoTransactions hugoTrans(*ctx->getTab());
UtilTransactions utilTrans(*ctx->getTab());
while (i<loops && result == NDBT_OK) {
g_info << i << ": ";
int timeout = 120;
// Test that the single user mode api can do everything
CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0);
CHECK(restarter.waitClusterSingleUser(timeout) == 0);
CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0);
CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
CHECK(count == records);
CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
CHECK(count == (records/2));
CHECK(utilTrans.clearTable(pNdb, records/2) == 0);
CHECK(restarter.exitSingleUserMode() == 0);
CHECK(restarter.waitClusterStarted(timeout) == 0);
// Test create index in single user mode
CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0);
CHECK(restarter.waitClusterSingleUser(timeout) == 0);
CHECK(create_index_on_pk(pNdb, tabName) == 0);
CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0);
CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
CHECK(count == records);
CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
CHECK(drop_index_on_pk(pNdb, tabName) == 0);
CHECK(restarter.exitSingleUserMode() == 0);
CHECK(restarter.waitClusterStarted(timeout) == 0);
// Test recreate index in single user mode
CHECK(create_index_on_pk(pNdb, tabName) == 0);
CHECK(hugoTrans.loadTable(pNdb, records, 128) == 0);
CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
CHECK(restarter.enterSingleUserMode(pNdb->getNodeId()) == 0);
CHECK(restarter.waitClusterSingleUser(timeout) == 0);
CHECK(drop_index_on_pk(pNdb, tabName) == 0);
CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
CHECK(create_index_on_pk(pNdb, tabName) == 0);
CHECK(restarter.exitSingleUserMode() == 0);
CHECK(restarter.waitClusterStarted(timeout) == 0);
CHECK(drop_index_on_pk(pNdb, tabName) == 0);
CHECK(utilTrans.clearTable(GETNDB(step), records) == 0);
ndbout << "Restarting cluster" << endl;
CHECK(restarter.restartAll() == 0);
CHECK(restarter.waitClusterStarted(timeout) == 0);
CHECK(pNdb->waitUntilReady(timeout) == 0);
i++;
}
return result;
}
int runRestoreBankAndVerify(NDBT_Context* ctx, NDBT_Step* step){
NdbRestarter restarter;
NdbBackup backup(GETNDB(step)->getNodeId()+1);
unsigned minBackupId = ctx->getProperty("MinBackupId");
unsigned maxBackupId = ctx->getProperty("MaxBackupId");
unsigned backupId = minBackupId;
int result = NDBT_OK;
int errSumAccounts = 0;
int errValidateGL = 0;
ndbout << " maxBackupId = " << maxBackupId << endl;
ndbout << " minBackupId = " << minBackupId << endl;
while (backupId <= maxBackupId){
// TEMPORARY FIX
// To erase all tables from cache(s)
// To be removed, maybe replaced by ndb.invalidate();
runDropTable(ctx,step);
{
Bank bank(ctx->m_cluster_connection);
if (bank.dropBank() != NDBT_OK){
result = NDBT_FAILED;
break;
}
}
// END TEMPORARY FIX
ndbout << "Performing restart" << endl;
if (restarter.restartAll(false) != 0)
return NDBT_FAILED;
if (restarter.waitClusterStarted() != 0)
return NDBT_FAILED;
ndbout << "Restoring backup " << backupId << endl;
if (backup.restore(backupId) == -1){
return NDBT_FAILED;
}
ndbout << "Backup " << backupId << " restored" << endl;
// Let bank verify
Bank bank(ctx->m_cluster_connection);
int wait = 0;
int yield = 1;
if (bank.performSumAccounts(wait, yield) != 0){
ndbout << "bank.performSumAccounts FAILED" << endl;
ndbout << " backupId = " << backupId << endl << endl;
result = NDBT_FAILED;
errSumAccounts++;
}
if (bank.performValidateAllGLs() != 0){
ndbout << "bank.performValidateAllGLs FAILED" << endl;
ndbout << " backupId = " << backupId << endl << endl;
result = NDBT_FAILED;
errValidateGL++;
}
backupId++;
}
if (result != NDBT_OK){
ndbout << "Verification of backup failed" << endl
<< " errValidateGL="<<errValidateGL<<endl
<< " errSumAccounts="<<errSumAccounts<<endl << endl;
}
return result;
}
