/* Producer thread */
int runV2MultiWait_Producer(NDBT_Context* ctx, NDBT_Step* step,
int thd_id, int nthreads)
{
int records = ctx->getNumRecords();
HugoOperations hugoOps(*ctx->getTab());
/* For three threads (2 producers + 1 consumer) we loop 0-7.
producer 0 is slow if (loop & 1)
producer 1 is slow if (loop & 2)
consumer is slow if (loop & 4)
*/
for (int loop = 0; loop < V2_NLOOPS; loop++)
{
ctx->getPropertyWait("LOOP", loop+1);
bool slow = loop & (thd_id+1);
for (int j=0; j < records; j++)
{
if(j % nthreads == thd_id)
{
Ndb* ndb = global_ndb_pool->getNdb();
NdbTransaction* trans = ndb->startTransaction();
check(trans != NULL, (*ndb));
ndb->setCustomData(trans);
NdbOperation* readOp = trans->getNdbOperation(ctx->getTab());
check(readOp != NULL, (*trans));
check(readOp->readTuple() == 0, (*readOp));
check(hugoOps.equalForRow(readOp, j) == 0, hugoOps);
/* Read all other cols */
for (int k=0; k < ctx->getTab()->getNoOfColumns(); k++)
{
check(readOp->getValue(ctx->getTab()->getColumn(k)) != NULL,
(*readOp));
}
trans->executeAsynchPrepare(NdbTransaction::Commit,
NULL,
NULL,
NdbOperation::AbortOnError);
ndb->sendPreparedTransactions();
global_poll_group->push(ndb);
if(slow)
{
int tm = myRandom48(3) * myRandom48(3);
if(tm) NdbSleep_MilliSleep(tm);
}
}
}
}
return NDBT_OK;
}
extern "C" void* NdbThreadFuncUpdate(void* pArg)
{
myRandom48Init((long int)NdbTick_CurrentMillisecond());
unsigned nSucc = 0;
unsigned nFail = 0;
Ndb* pNdb = NULL ;
pNdb = new Ndb("TEST_DB");
VerifyMethodInt(pNdb, init());
VerifyMethodInt(pNdb, waitUntilReady());
while(NdbMutex_Trylock(g_pNdbMutex)) {
Uint32 nWarehouse = myRandom48(g_nWarehouseCount);
NdbConnection* pNdbConnection = NULL ;
VerifyMethodPtr(pNdbConnection, pNdb, startTransaction());
CHK_TR(pNdbConnection) ; // epaulsa
NdbOperation* pNdbOperationW = NULL ;
VerifyMethodPtr(pNdbOperationW, pNdbConnection, getNdbOperation(c_szWarehouse));
VerifyMethodInt(pNdbOperationW, interpretedUpdateTuple());
VerifyMethodInt(pNdbOperationW, equal(c_szWarehouseNumber, nWarehouse));
VerifyMethodInt(pNdbOperationW, incValue(c_szWarehouseCount, Uint32(1)));
Uint32 nWarehouseSum = 0;
for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) {
NdbOperation* pNdbOperationD = NULL ;
VerifyMethodPtr(pNdbOperationD, pNdbConnection, getNdbOperation(c_szDistrict));
VerifyMethodInt(pNdbOperationD, interpretedUpdateTuple());
VerifyMethodInt(pNdbOperationD, equal(c_szDistrictWarehouseNumber, nWarehouse));
VerifyMethodInt(pNdbOperationD, equal(c_szDistrictNumber, nDistrict));
VerifyMethodInt(pNdbOperationD, incValue(c_szDistrictCount, Uint32(1)));
Uint32 nDistrictSum = myRandom48(100);
nWarehouseSum += nDistrictSum;
VerifyMethodInt(pNdbOperationD, setValue(c_szDistrictSum, nDistrictSum));
}
VerifyMethodInt(pNdbOperationW, setValue(c_szWarehouseSum, nWarehouseSum));
int iExec = pNdbConnection->execute(Commit);
int iError = pNdbConnection->getNdbError().code;
if(iExec<0 && iError!=0 && iError!=266 && iError!=626) {
ReportMethodInt(iExec, pNdbConnection, "pNdbConnection", "execute(Commit)", __FILE__, __LINE__);
}
if(iExec==0) {
++nSucc;
} else {
++nFail;
}
VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection));
}
ndbout << "update: " << nSucc << " succeeded, " << nFail << " failed " << endl;
NdbMutex_Unlock(g_pNdbMutex);
delete pNdb;
pNdb = NULL ;
return NULL;
}
static
void
doTransaction_T4(Ndb * pNDB, ThreadData * td, int async)
{
/*----------------*/
/* Init arguments */
/*----------------*/
getRandomSubscriberNumber(td->transactionData.number);
getRandomServerId(&td->transactionData.server_id);
td->transactionData.server_bit = (1 << td->transactionData.server_id);
td->transactionData.do_rollback =
getNextRandom(&td->generator.rollbackSequenceT4);
#if 0
memset(td->transactionData.session_details,
myRandom48(26)+'A', SESSION_DETAILS_LENGTH);
#endif
td->transactionData.session_details[SESSION_DETAILS_LENGTH] = 0;
/*-----------------*/
/* Run transaction */
/*-----------------*/
td->runState = Running;
td->generator.transactions[3].startLatency();
start_T4(pNDB, td, async);
}
int fiftyPercentStopAndWait(F_ARGS){
int nodes[MAX_NDB_NODES];
int numNodes = get50PercentOfNodes(_restarter, nodes);
// Stop the nodes, with nostart and abort
for (int i = 0; i < numNodes; i++){
g_info << "Stopping node "<<nodes[i] << endl;
int res = _restarter.restartOneDbNode(nodes[i], false, true, true);
CHECK(res == 0, "Could not stop node: "<< nodes[i]);
}
CHECK(_restarter.waitNodesNoStart(nodes, numNodes) == 0,
"waitNodesNoStart");
// Create random value, max 120 secs
int max = 120;
int seconds = (myRandom48(max)) + 1;
g_info << "Waiting for " << seconds << "(" << max
<< ") secs " << endl;
NdbSleep_SecSleep(seconds);
// Restart graceful
CHECK(_restarter.restartAll() == 0,
"Could not restart all nodes");
g_info << _restart->m_name << endl;
return NDBT_OK;
}
int stopOnError(F_ARGS){
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
do {
g_info << _restart->m_name << ": node = " << nodeId
<< endl;
CHECK(_restarter.waitClusterStarted(300) == 0,
"waitClusterStarted failed");
int val = DumpStateOrd::NdbcntrTestStopOnError;
CHECK(_restarter.dumpStateOneNode(nodeId, &val, 1) == 0,
"failed to set NdbcntrTestStopOnError");
NdbSleep_SecSleep(3);
CHECK(_restarter.waitClusterStarted(300) == 0,
"waitClusterStarted failed");
} while (false);
return NDBT_OK;
}
int main(int arg, char** argv)
{
/* Test SectionReader
* -------------------
* To run this code :
* cd storage/ndb/src/kernel/vm
* make testSectionReader
* ./testSectionReader
*
* Will print "OK" in success case and return 0
*/
g_sectionSegmentPool.setSize(1024);
printf("g_sectionSegmentPool size is %u\n",
g_sectionSegmentPool.getSize());
const Uint32 Iterations= 2000;
const Uint32 Sections= 5;
Uint32 sizes[ Sections ];
Uint32 iVals[ Sections ];
for (Uint32 t=0; t < Iterations; t++)
{
for (Uint32 i=0; i<Sections; i++)
{
Uint32 available= g_sectionSegmentPool.getNoOfFree();
sizes[i] = available ?
myRandom48(SectionSegment::DataLength *
available)
: 0;
//if (0 == (sizes[i] % 60))
// printf("Iteration %u, section %u, allocating %u words\n",
// t, i, sizes[i]);
if (t % 100 == 0)
if (i==0)
printf("\nIteration %u", t);
if (sizes[i] > 0)
{
iVals[i]= createSection(&g_sectionSegmentPool, sizes[i]);
VERIFY(testSR(iVals[i], &g_sectionSegmentPool, sizes[i]) == 0);
}
else
iVals[i]= RNIL;
}
for (Uint32 i=0; i < Sections; i++)
{
if (sizes[i] > 0)
freeSection(&g_sectionSegmentPool, iVals[i]);
}
}
printf("\nOK\n");
return 0;
}
int getRandomNodeId(NdbRestarter& _restarter) {
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
return nodeId;
}
static void getRandomSubscriberNumber(SubscriberNumber number)
{
uint32 tmp;
char sbuf[SUBSCRIBER_NUMBER_LENGTH + 1];
tmp = myRandom48(NO_OF_SUBSCRIBERS);
sprintf(sbuf, "%.*d", SUBSCRIBER_NUMBER_LENGTH, tmp);
memcpy(number, sbuf, SUBSCRIBER_NUMBER_LENGTH);
}
static void clearTransaction(TransactionDefinition *trans)
{
trans->count = 0;
trans->branchExecuted = 0;
trans->rollbackExecuted = 0;
trans->latencyCounter = myRandom48(127);
trans->latency.reset();
}
int restartRandomNodeInitial(F_ARGS){
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
g_info << _restart->m_name << ": node = "<<nodeId << endl;
CHECK(_restarter.restartOneDbNode(nodeId, true) == 0,
"Could not restart node "<<nodeId);
return NDBT_OK;
}
int restartRandomNodeError(F_ARGS){
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
ndbout << _restart->m_name << ": node = "<<nodeId << endl;
CHECK(_restarter.insertErrorInNode(nodeId, 9999) == 0,
"Could not restart node "<<nodeId);
return NDBT_OK;
}
int runBuddyTransNoTimeout(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
int stepNo = step->getStepNo();
int maxSleep = (int)(TIMEOUT * 0.3);
ndbout << "TransactionInactiveTimeout="<< TIMEOUT
<< ", maxSleep="<<maxSleep<<endl;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
for (int l = 1; l < loops && result == NDBT_OK; l++){
do{
// Start an insert trans
CHECK(hugoOps.startTransaction(pNdb) == 0);
int recordNo = records + (stepNo*loops) + l;
CHECK(hugoOps.pkInsertRecord(pNdb, recordNo) == 0);
CHECK(hugoOps.execute_NoCommit(pNdb) == 0);
int remain = maxSleep;
for (int i = 0; i < 3; i++)
{
NdbTransaction* pTrans = hugoOps.getTransaction();
// Perform buddy scan reads
NdbScanOperation* pOp = pTrans->getNdbScanOperation(ctx->getTab());
CHECK(pOp != 0);
CHECK(pOp->readTuples(NdbOperation::LM_Read, 0, 0, 1) == 0);
CHECK(pTrans->execute(NoCommit) == 0);
while(pOp->nextResult() == 0);
int sleep = myRandom48(remain);
remain = remain - sleep + 1;
ndbout << "Sleeping for " << sleep << " milliseconds" << endl;
NdbSleep_MilliSleep(sleep);
}
// Expect that transaction has NOT timed-out
CHECK(hugoOps.execute_Commit(pNdb) == 0);
} while(false);
hugoOps.closeTransaction(pNdb);
}
return result;
}
int twoNodeFailure(F_ARGS){
myRandom48Init((long)NdbTick_CurrentMillisecond());
int randomId = myRandom48(_restarter.getNumDbNodes());
int n[2];
n[0] = _restarter.getDbNodeId(randomId);
n[1] = _restarter.getRandomNodeOtherNodeGroup(n[0], rand());
g_info << _restart->m_name << ": node = "<< n[0] << endl;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
CHECK(_restarter.dumpStateOneNode(n[0], val2, 2) == 0,
"Failed to dump");
CHECK(_restarter.dumpStateOneNode(n[1], val2, 2) == 0,
"Failed to dump");
CHECK(_restarter.insertErrorInNode(n[0], 9999) == 0,
"Could not restart node "<< n[0]);
// Create random value, max 3 secs
int max = 3000;
int ms = (myRandom48(max)) + 1;
g_info << "Waiting for " << ms << "(" << max
<< ") ms " << endl;
NdbSleep_MilliSleep(ms);
g_info << _restart->m_name << ": node = "<< n[1] << endl;
CHECK(_restarter.insertErrorInNode(n[1], 9999) == 0,
"Could not restart node "<< n[1]);
CHECK(_restarter.waitNodesNoStart(n, 2) == 0,
"Failed to wait nostart");
_restarter.startNodes(n, 2);
return NDBT_OK;
}
int runDeadlockTimeoutTrans(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int stepNo = step->getStepNo();
Uint32 deadlock_timeout;
NdbConfig conf(GETNDB(step)->getNodeId()+1);
unsigned int nodeId = conf.getMasterNodeId();
if (!conf.getProperty(nodeId,
NODE_TYPE_DB,
CFG_DB_TRANSACTION_DEADLOCK_TIMEOUT,
&deadlock_timeout)){
return NDBT_FAILED;
}
int do_sleep = (int)(deadlock_timeout * 0.5);
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
for (int l = 0; l < loops && result == NDBT_OK; l++){
do{
// Commit transaction
CHECK(hugoOps.startTransaction(pNdb) == 0);
CHECK(hugoOps.pkReadRecord(pNdb, stepNo) == 0);
CHECK(hugoOps.execute_NoCommit(pNdb) == 0);
int sleep = deadlock_timeout * 1.5 + myRandom48(do_sleep);
ndbout << "Sleeping for " << sleep << " milliseconds" << endl;
NdbSleep_MilliSleep(sleep);
// Expect that transaction has NOT timed-out
CHECK(hugoOps.execute_Commit(pNdb) == 0);
} while(false);
hugoOps.closeTransaction(pNdb);
}
return result;
}
int twoMasterNodeFailure(F_ARGS){
int n[2];
n[0] = _restarter.getMasterNodeId();
n[1] = n[0];
do {
n[1] = _restarter.getNextMasterNodeId(n[1]);
} while(_restarter.getNodeGroup(n[0]) == _restarter.getNodeGroup(n[1]));
g_info << _restart->m_name << ": ";
g_info << "node0 = "<< n[0] << "(" << _restarter.getNodeGroup(n[0]) << ") ";
g_info << "node1 = "<< n[1] << "(" << _restarter.getNodeGroup(n[1]) << ") ";
g_info << endl;
int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
CHECK(_restarter.dumpStateOneNode(n[0], val2, 2) == 0,
"Failed to dump");
CHECK(_restarter.dumpStateOneNode(n[1], val2, 2) == 0,
"Failed to dump");
CHECK(_restarter.insertErrorInNode(n[0], 9999) == 0,
"Could not restart node "<< n[0]);
// Create random value, max 3 secs
int max = 3000;
int ms = (myRandom48(max)) + 1;
g_info << "Waiting for " << ms << "(" << max
<< ") ms " << endl;
NdbSleep_MilliSleep(ms);
g_info << _restart->m_name << ": node = "<< n[1] << endl;
CHECK(_restarter.insertErrorInNode(n[1], 9999) == 0,
"Could not restart node "<< n[1]);
CHECK(_restarter.waitNodesNoStart(n, 2) == 0,
"Failed to wait nostart");
_restarter.startNodes(n, 2);
return NDBT_OK;
}
/* Consumer */
int runV2MultiWait_WaitPop_Thread(NDBT_Context* ctx, NDBT_Step* step)
{
static int iter = 0; // keeps incrementing when test case is repeated
int records = ctx->getNumRecords();
const char * d[5] = { " fast"," slow"," slow",""," slow" };
const int timeout[3] = { 100, 1, 0 };
const int pct_wait[9] = { 0,0,0,50,50,50,100,100,100 };
for (int loop = 0; loop < V2_NLOOPS; loop++, iter++)
{
ctx->incProperty("LOOP");
ndbout << "V2 test: " << d[loop&1] << d[loop&2] << d[loop&4];
ndbout << " " << timeout[iter%3] << "/" << pct_wait[iter%9] << endl;
bool slow = loop & 4;
int nrec = 0;
while(nrec < records)
{
/* Occasionally check with no timeout */
global_poll_group->wait(timeout[iter%3], pct_wait[iter%9]);
Ndb * ndb = global_poll_group->pop();
while(ndb)
{
check(ndb->pollNdb(0, 1) != 0, (*ndb));
nrec++;
NdbTransaction *tx = (NdbTransaction *) ndb->getCustomData();
tx->close();
global_ndb_pool->recycleNdb(ndb);
ndb = global_poll_group->pop();
}
if(slow)
{
NdbSleep_MilliSleep(myRandom48(6));
}
}
}
ctx->stopTest();
global_ndb_pool->closeAll();
return NDBT_OK;
}
static
void
doTransaction_T1(Ndb * pNDB, ThreadData * td, int async)
{
/*----------------*/
/* Init arguments */
/*----------------*/
getRandomSubscriberNumber(td->transactionData.number);
getRandomChangedBy(td->transactionData.changed_by);
BaseString::snprintf(td->transactionData.changed_time,
sizeof(td->transactionData.changed_time),
"%ld - %d", td->changedTime++, myRandom48(65536*1024));
//getRandomChangedTime(td->transactionData.changed_time);
td->transactionData.location = td->transactionData.changed_by[0];
/*-----------------*/
/* Run transaction */
/*-----------------*/
td->runState = Running;
td->generator.transactions[0].startLatency();
start_T1(pNDB, td, async);
}
int runDontTimeoutTrans(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int stepNo = step->getStepNo();
int timeout = ctx->getProperty("TransactionInactiveTimeout",TIMEOUT);
int maxSleep = (int)(timeout * 0.5);
ndbout << "TransactionInactiveTimeout="<< timeout
<< ", maxSleep="<<maxSleep<<endl;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
for (int l = 0; l < loops && result == NDBT_OK; l++){
do{
// Commit transaction
CHECK(hugoOps.startTransaction(pNdb) == 0);
CHECK(hugoOps.pkReadRecord(pNdb, stepNo) == 0);
CHECK(hugoOps.execute_NoCommit(pNdb) == 0);
int sleep = myRandom48(maxSleep);
ndbout << "Sleeping for " << sleep << " milliseconds" << endl;
NdbSleep_MilliSleep(sleep);
// Expect that transaction has NOT timed-out
CHECK(hugoOps.execute_Commit(pNdb) == 0);
} while(false);
hugoOps.closeTransaction(pNdb);
}
return result;
}
static int
runBug14702377(NDBT_Context* ctx, NDBT_Step* step)
{
Ndb* pNdb = GETNDB(step);
NdbDictionary::Dictionary * pDict = pNdb->getDictionary();
int records = ctx->getNumRecords();
int result = NDBT_OK;
while (ctx->getProperty("HalfStartedDone", (Uint32)0) == 0)
{
ndbout << "Wait for half started..." << endl;
NdbSleep_SecSleep(15);
}
ndbout << "Got half started" << endl;
while (1)
{
require(table_list.size() == 1);
const char* tabname = table_list[0].c_str();
const NdbDictionary::Table* tab = 0;
CHK2((tab = pDict->getTable(tabname)) != 0,
tabname << ": " << pDict->getNdbError());
const int ncol = tab->getNoOfColumns();
{
HugoTransactions trans(*tab);
CHK2(trans.loadTable(pNdb, records) == 0, trans.getNdbError());
}
for (int r = 0; r < records; r++)
{
// with 1000 records will surely hit bug case
const int lm = myRandom48(4); // 2
const int nval = myRandom48(ncol + 1); // most
const bool exist = myRandom48(2); // false
NdbTransaction* pTx = 0;
NdbOperation* pOp = 0;
CHK2((pTx = pNdb->startTransaction()) != 0,
pNdb->getNdbError());
CHK2((pOp = pTx->getNdbOperation(tab)) != 0,
pTx->getNdbError());
CHK2((pOp->readTuple((NdbOperation::LockMode)lm)) == 0,
pOp->getNdbError());
for (int id = 0; id <= 0; id++)
{
const NdbDictionary::Column* c = tab->getColumn(id);
require(c != 0 && c->getPrimaryKey() &&
c->getType() == NdbDictionary::Column::Unsigned);
Uint32 val = myRandom48(records);
if (!exist)
val = 0xaaaa0000 + myRandom48(0xffff + 1);
const char* valp = (const char*)&val;
CHK2(pOp->equal(id, valp) == 0, pOp->getNdbError());
}
CHK2(result == NDBT_OK, "failed");
for (int id = 0; id < nval; id++)
{
const NdbDictionary::Column* c = tab->getColumn(id);
require(c != 0 && (id == 0 || !c->getPrimaryKey()));
CHK2(pOp->getValue(id) != 0, pOp->getNdbError());
}
CHK2(result == NDBT_OK, "failed");
char info1[200];
sprintf(info1, "lm=%d nval=%d exist=%d",
lm, nval, exist);
g_info << "PK read T1 exec: " << info1 << endl;
Uint64 t1 = NdbTick_CurrentMillisecond();
int ret = pTx->execute(NdbTransaction::NoCommit);
Uint64 t2 = NdbTick_CurrentMillisecond();
int msec = (int)(t2-t1);
const NdbError& txerr = pTx->getNdbError();
const NdbError& operr = pOp->getNdbError();
char info2[200];
sprintf(info2, "%s msec=%d ret=%d txerr=%d operr=%d",
info1, msec, ret, txerr.code, operr.code);
g_info << "PK read T1 done: " << info2 << endl;
if (ret == 0 && txerr.code == 0 && operr.code == 0)
{
CHK2(exist, "row should not be found: " << info2);
}
else if (ret == 0 && txerr.code == 626 && operr.code == 626)
{
CHK2(!exist, "row should be found: " << info2);
}
else if (txerr.status == NdbError::TemporaryError)
{
g_err << "PK read T1 temporary error (tx): " << info2 << endl;
NdbSleep_MilliSleep(50);
}
else if (operr.status == NdbError::TemporaryError)
{
g_err << "PK read T1 temporary error (op): " << info2 << endl;
NdbSleep_MilliSleep(50);
}
else
{
// gets 4012 before bugfix
CHK2(false, "unexpected error: " << info2);
}
pNdb->closeTransaction(pTx);
pTx = 0;
}
break;
}
g_err << "Clear half started hold..." << endl;
ctx->setProperty("HalfStartedHold", (Uint32)0);
return result;
}
int runTimeoutTrans2(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int stepNo = step->getStepNo();
int mul1 = ctx->getProperty("Op1", (Uint32)0);
int mul2 = ctx->getProperty("Op2", (Uint32)0);
int records = ctx->getNumRecords();
int timeout = ctx->getProperty("TransactionInactiveTimeout",TIMEOUT);
int minSleep = (int)(timeout * 1.5);
int maxSleep = timeout * 2;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
for (int l = 0; l<loops && !ctx->isTestStopped() && result == NDBT_OK; l++){
int op1 = 0 + (l + stepNo) * mul1;
int op2 = 0 + (l + stepNo) * mul2;
op1 = (op1 % 5);
op2 = (op2 % 5);
ndbout << stepNo << ": TransactionInactiveTimeout="<< timeout
<< ", minSleep="<<minSleep
<< ", maxSleep="<<maxSleep
<< ", op1=" << op1
<< ", op2=" << op2 << endl;;
do{
// Commit transaction
CHECK(hugoOps.startTransaction(pNdb) == 0);
switch(op1){
case 0:
break;
case 1:
if(hugoOps.pkReadRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 2:
if(hugoOps.pkUpdateRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 3:
if(hugoOps.pkDeleteRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 4:
if(hugoOps.pkInsertRecord(pNdb, stepNo+records+l) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
}
if(result != NDBT_OK)
break;
int res = hugoOps.execute_NoCommit(pNdb);
if(res != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
int sleep = minSleep + myRandom48(maxSleep-minSleep);
ndbout << stepNo << ": Sleeping for "<< sleep << " milliseconds" << endl;
NdbSleep_MilliSleep(sleep);
switch(op2){
case 0:
break;
case 1:
if(hugoOps.pkReadRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 2:
if(hugoOps.pkUpdateRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 3:
if(hugoOps.pkDeleteRecord(pNdb, stepNo) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
case 4:
if(hugoOps.pkInsertRecord(pNdb, stepNo+2*records+l) != 0){
g_err << stepNo << ": Fail" << __LINE__ << endl;
result = NDBT_FAILED; break;
}
break;
}
// Expect that transaction has timed-out
res = hugoOps.execute_Commit(pNdb);
if(op1 != 0 && res != 266){
g_err << stepNo << ": Fail: " << res << "!= 237, op1="
<< op1 << ", op2=" << op2 << endl;
result = NDBT_FAILED; break;
}
} while(false);
hugoOps.closeTransaction(pNdb);
}
return result;
}
int testSR(Uint32 iVal, SectionSegmentPool* ssp, Uint32 len)
{
SectionReader srStepPeek(iVal, *ssp);
SectionReader srGetWord(iVal, *ssp);
SectionReader srPosSource(iVal, *ssp);
SectionReader srPosDest(iVal, *ssp);
SectionReader srPtrWord(iVal, *ssp);
VERIFY(srStepPeek.getSize() == len);
/* Reset the section readers at a random position */
const Uint32 noResetPos= 9999999;
Uint32 resetAt= len> 10 ? myRandom48(len) : noResetPos;
/* Read from the section readers, 1 word at a time */
for (Uint32 i=0; i < len; i++)
{
Uint32 peekWord;
Uint32 getWord;
const Uint32* ptrWord;
Uint32 ptrReadSize;
/* Check that peek, getWord and getWordsPtr return
* the same, correct value
*/
VERIFY(srStepPeek.peekWord(&peekWord));
if (i < (len -1))
VERIFY(srStepPeek.step(1));
VERIFY(srGetWord.getWord(&getWord));
VERIFY(srPtrWord.getWordsPtr(1, ptrWord, ptrReadSize));
VERIFY(ptrReadSize == 1);
//printf("PeekWord=%u, i=%u\n",
// peekWord, i);
VERIFY(peekWord == i);
VERIFY(peekWord == getWord);
VERIFY(peekWord == *ptrWord);
/* Check that one sectionReader with it's position
* set from the first returns the same, correct word
*/
SectionReader::PosInfo p= srPosSource.getPos();
srPosDest.setPos(p);
Uint32 srcWord, destWord;
VERIFY(srPosSource.getWord(&srcWord));
VERIFY(srPosDest.getWord(&destWord));
VERIFY(srcWord == peekWord);
VERIFY(srcWord == destWord);
/* Reset the readers */
if (i == resetAt)
{
//printf("Resetting\n");
resetAt= noResetPos;
i= (Uint32) -1;
srStepPeek.reset();
srGetWord.reset();
srPosSource.reset();
srPosDest.reset();
srPtrWord.reset();
}
else
{
if ((myRandom48(400) == 1) &&
(i < len -1))
{
/* Step all readers forward by some amount */
Uint32 stepSize= myRandom48((len - i) -1 );
//printf("Stepping %u words\n", stepSize);
VERIFY(srStepPeek.step(stepSize));
VERIFY(srGetWord.step(stepSize));
VERIFY(srPosSource.step(stepSize));
VERIFY(srPosDest.step(stepSize));
VERIFY(srPtrWord.step(stepSize));
i+= stepSize;
}
}
}
/* Check that there's nothing left in any reader */
VERIFY(!srStepPeek.step(1));
VERIFY(!srGetWord.step(1));
VERIFY(!srPosSource.step(1));
VERIFY(!srPosDest.step(1));
VERIFY(!srPtrWord.step(1));
srStepPeek.reset();
srGetWord.reset();
srPosSource.reset();
srPosDest.reset();
srPtrWord.reset();
/* Now read larger chunks of words */
Uint32 pos= 0;
Uint32* buffer= (Uint32*) malloc(len * 4);
VERIFY(buffer != NULL);
while (pos < len)
{
const Uint32 remain= len-pos;
const Uint32 readSize= remain == 1 ? 1 : myRandom48(remain);
//printf("Pos=%u Len=%u readSize=%u \n", pos, len, readSize);
/* Check that peek + step get the correct words */
VERIFY(srStepPeek.peekWords(buffer, readSize));
if (len > pos + readSize)
{
VERIFY(srStepPeek.step(readSize));
}
else
VERIFY(srStepPeek.step((len - pos) - 1));
VERIFY(checkBuffer(buffer, pos, readSize) == 0);
/* Check that getWords gets the correct words */
VERIFY(srGetWord.getWords(buffer, readSize));
VERIFY(checkBuffer(buffer, pos, readSize) == 0);
/* Check that using getPos + setPos gets the correct words */
VERIFY(srPosDest.setPos(srPosSource.getPos()));
VERIFY(srPosSource.getWords(buffer, readSize));
VERIFY(checkBuffer(buffer, pos, readSize) == 0);
VERIFY(srPosDest.getWords(buffer, readSize));
VERIFY(checkBuffer(buffer, pos, readSize) == 0);
/* Check that getWordsPtr gets the correct words */
Uint32 ptrWordsRead= 0;
//printf("Reading from ptr\n");
while (ptrWordsRead < readSize)
{
const Uint32* ptr= NULL;
Uint32 readLen;
VERIFY(srPtrWord.getWordsPtr((readSize - ptrWordsRead),
ptr,
readLen));
VERIFY(readLen <= readSize);
//printf("Read %u words, from pos %u, offset %u\n",
// readLen, pos, ptrWordsRead);
VERIFY(checkBuffer(ptr, pos+ ptrWordsRead, readLen) == 0);
ptrWordsRead+= readLen;
}
pos += readSize;
}
/* Check that there's no more words in any reader */
VERIFY(!srStepPeek.step(1));
VERIFY(!srGetWord.step(1));
VERIFY(!srPosSource.step(1));
VERIFY(!srPosDest.step(1));
VERIFY(!srPtrWord.step(1));
/* Verify that ptr-fetch variants do not fetch beyond
* the length, even if we ask for more
*/
srPtrWord.reset();
Uint32 readWords= 0;
while (readWords < len)
{
const Uint32* readPtr;
Uint32 wordsRead= 0;
VERIFY(srPtrWord.getWordsPtr(20, readPtr, wordsRead));
readWords+= wordsRead;
VERIFY(readWords <= len);
}
free(buffer);
return 0;
}
static void getRandomChangedBy(ChangedBy changedBy)
{
memset(changedBy, myRandom48(26)+'A', CHANGED_BY_LENGTH);
changedBy[CHANGED_BY_LENGTH] = 0;
}
int restartNFDuringNR(F_ARGS safety){
myRandom48Init((long)NdbTick_CurrentMillisecond());
int i;
const int sz = sizeof(NFDuringNR_codes)/sizeof(NFDuringNR_codes[0]);
for(i = 0; i<sz && !ctx->closeToTimeout(safety); i++){
int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
int error = NFDuringNR_codes[i];
g_err << _restart->m_name << ": node = " << nodeId
<< " error code = " << error << endl;
CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0,
"Could not restart node "<< nodeId);
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 } ;
CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0,
"failed to set RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(nodeId, error) == 0,
"failed to set error insert");
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
NdbSleep_SecSleep(3);
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
CHECK(_restarter.waitNodesStarted(&nodeId, 1) == 0,
"waitNodesStarted failed");
}
return NDBT_OK;
if(_restarter.getNumDbNodes() < 4)
return NDBT_OK;
#ifdef NDB_USE_GET_ENV
char buf[256];
if(NdbEnv_GetEnv("USER", buf, 256) == 0 || strcmp(buf, "ejonore") != 0)
return NDBT_OK;
for(i = 0; i<sz && !ctx->isTestStopped() && !ctx->closeToTimeout(safety);i++){
const int randomId = myRandom48(_restarter.getNumDbNodes());
int nodeId = _restarter.getDbNodeId(randomId);
const int error = NFDuringNR_codes[i];
const int masterNodeId = _restarter.getMasterNodeId();
CHECK(masterNodeId > 0, "getMasterNodeId failed");
int crashNodeId = 0;
do {
int rand = myRandom48(1000);
crashNodeId = _restarter.getRandomNodeOtherNodeGroup(nodeId, rand);
} while(crashNodeId == masterNodeId);
CHECK(crashNodeId > 0, "getMasterNodeId failed");
g_info << _restart->m_name << " restarting node = " << nodeId
<< " error code = " << error
<< " crash node = " << crashNodeId << endl;
CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0,
"Could not restart node "<< nodeId);
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
CHECK(_restarter.dumpStateOneNode(crashNodeId, val, 2) == 0,
"failed to set RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(crashNodeId, error) == 0,
"failed to set error insert");
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
}
#endif
return NDBT_OK;
}
static void getRandomChangedTime(ChangedTime changedTime)
{
memset(changedTime, myRandom48(26)+'A', CHANGED_TIME_LENGTH);
changedTime[CHANGED_TIME_LENGTH] = 0;
}
static void getRandomServerId(ServerId *serverId)
{
*serverId = myRandom48(NO_OF_SERVERS);
}
int
NdbBackup::NF(NdbRestarter& _restarter, int *NFDuringBackup_codes, const int sz, bool onMaster){
int nNodes = _restarter.getNumDbNodes();
{
if(nNodes == 1)
return NDBT_OK;
int nodeId = _restarter.getMasterNodeId();
CHECK(_restarter.restartOneDbNode(nodeId, false, true, true) == 0,
"Could not restart node "<< nodeId);
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
}
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
myRandom48Init(NdbTick_CurrentMillisecond());
for(int i = 0; i<sz; i++){
int error = NFDuringBackup_codes[i];
unsigned int backupId;
const int masterNodeId = _restarter.getMasterNodeId();
CHECK(masterNodeId > 0, "getMasterNodeId failed");
int nodeId;
nodeId = masterNodeId;
if (!onMaster) {
int randomId;
while (nodeId == masterNodeId) {
randomId = myRandom48(nNodes);
nodeId = _restarter.getDbNodeId(randomId);
}
}
g_err << "NdbBackup::NF node = " << nodeId
<< " error code = " << error << " masterNodeId = "
<< masterNodeId << endl;
int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
CHECK(_restarter.dumpStateOneNode(nodeId, val, 2) == 0,
"failed to set RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(nodeId, error) == 0,
"failed to set error insert");
g_info << "error inserted" << endl;
NdbSleep_SecSleep(1);
g_info << "starting backup" << endl;
int r = start(backupId);
g_info << "r = " << r
<< " (which should fail) started with id = " << backupId << endl;
if (r == 0) {
g_err << "Backup should have failed on error_insertion " << error << endl
<< "Master = " << masterNodeId << "Node = " << nodeId << endl;
return NDBT_FAILED;
}
CHECK(_restarter.waitNodesNoStart(&nodeId, 1) == 0,
"waitNodesNoStart failed");
g_info << "number of nodes running " << _restarter.getNumDbNodes() << endl;
if (_restarter.getNumDbNodes() != nNodes) {
g_err << "Failure: cluster not up" << endl;
return NDBT_FAILED;
}
g_info << "starting new backup" << endl;
CHECK(start(backupId) == 0,
"failed to start backup");
g_info << "(which should succeed) started with id = " << backupId << endl;
g_info << "starting node" << endl;
CHECK(_restarter.startNodes(&nodeId, 1) == 0,
"failed to start node");
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
g_info << "node started" << endl;
int val2[] = { 24, 2424 };
CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0,
"failed to check backup resources RestartOnErrorInsert");
CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0,
"failed to set error insert");
NdbSleep_SecSleep(1);
}
return NDBT_OK;
}
const NdbRestarts::NdbErrorInsert* NdbRestarts::getRandomError(){
int randomId = myRandom48(m_NoOfErrors);
return &m_errors[randomId];
}
extern "C" void* NdbThreadFuncRead(void* pArg)
{
myRandom48Init((long int)NdbTick_CurrentMillisecond());
unsigned nSucc = 0;
unsigned nFail = 0;
NdbRecAttr** ppNdbRecAttrDSum = new NdbRecAttr*[g_nDistrictPerWarehouse];
NdbRecAttr** ppNdbRecAttrDCnt = new NdbRecAttr*[g_nDistrictPerWarehouse];
Ndb* pNdb = NULL ;
pNdb = new Ndb("TEST_DB");
VerifyMethodInt(pNdb, init());
VerifyMethodInt(pNdb, waitUntilReady());
while(NdbMutex_Trylock(g_pNdbMutex)) {
Uint32 nWarehouse = myRandom48(g_nWarehouseCount);
NdbConnection* pNdbConnection = NULL ;
VerifyMethodPtr(pNdbConnection, pNdb, startTransaction());
CHK_TR(pNdbConnection) ; // epaulsa
NdbOperation* pNdbOperationW = NULL ;
VerifyMethodPtr(pNdbOperationW, pNdbConnection, getNdbOperation(c_szWarehouse));
VerifyMethodInt(pNdbOperationW, readTuple());
VerifyMethodInt(pNdbOperationW, equal(c_szWarehouseNumber, nWarehouse));
NdbRecAttr* pNdbRecAttrWSum;
VerifyMethodPtr(pNdbRecAttrWSum, pNdbOperationW, getValue(c_szWarehouseSum, 0));
NdbRecAttr* pNdbRecAttrWCnt;
VerifyMethodPtr(pNdbRecAttrWCnt, pNdbOperationW, getValue(c_szWarehouseCount, 0));
for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) {
NdbOperation* pNdbOperationD = NULL ;
VerifyMethodPtr(pNdbOperationD, pNdbConnection, getNdbOperation(c_szDistrict));
VerifyMethodInt(pNdbOperationD, readTuple());
VerifyMethodInt(pNdbOperationD, equal(c_szDistrictWarehouseNumber, nWarehouse));
VerifyMethodInt(pNdbOperationD, equal(c_szDistrictNumber, nDistrict));
VerifyMethodPtr(ppNdbRecAttrDSum[nDistrict], pNdbOperationD, getValue(c_szDistrictSum, 0));
VerifyMethodPtr(ppNdbRecAttrDCnt[nDistrict], pNdbOperationD, getValue(c_szDistrictCount, 0));
}
int iExec = pNdbConnection->execute(Commit);
int iError = pNdbConnection->getNdbError().code;
if(iExec<0 && iError!=0 && iError!=266 && iError!=626) {
ReportMethodInt(iExec, pNdbConnection, "pNdbConnection", "execute(Commit)", __FILE__, __LINE__);
}
if(iExec==0) {
Uint32 nSum = 0;
Uint32 nCnt = 0;
for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) {
nSum += ppNdbRecAttrDSum[nDistrict]->u_32_value();
nCnt += ppNdbRecAttrDCnt[nDistrict]->u_32_value();
}
if(nSum!=pNdbRecAttrWSum->u_32_value()
|| nCnt!=g_nDistrictPerWarehouse*pNdbRecAttrWCnt->u_32_value()) {
ndbout << "INCONSISTENT!" << endl;
ndbout << "iExec==" << iExec << endl;
ndbout << "iError==" << iError << endl;
ndbout << endl;
ndbout << c_szWarehouseSum << "==" << pNdbRecAttrWSum->u_32_value() << ", ";
ndbout << c_szWarehouseCount << "==" << pNdbRecAttrWCnt->u_32_value() << endl;
ndbout << "nSum==" << nSum << ", nCnt=" << nCnt << endl;
for(Uint32 nDistrict=0; nDistrict<g_nDistrictPerWarehouse; ++nDistrict) {
ndbout << c_szDistrictSum << "[" << nDistrict << "]==" << ppNdbRecAttrDSum[nDistrict]->u_32_value() << ", ";
ndbout << c_szDistrictCount << "[" << nDistrict << "]==" << ppNdbRecAttrDCnt[nDistrict]->u_32_value() << endl;
}
VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection));
delete pNdb;
pNdb = NULL ;
delete[] ppNdbRecAttrDSum;
ppNdbRecAttrDSum = NULL ;
delete[] ppNdbRecAttrDCnt;
ppNdbRecAttrDCnt = NULL ;
NDBT_ProgramExit(NDBT_FAILED);
}
++nSucc;
} else {
++nFail;
}
VerifyMethodVoid(pNdb, closeTransaction(pNdbConnection));
}
ndbout << "read: " << nSucc << " succeeded, " << nFail << " failed " << endl;
NdbMutex_Unlock(g_pNdbMutex);
delete pNdb;
pNdb = NULL ;
delete[] ppNdbRecAttrDSum;
ppNdbRecAttrDSum = NULL ;
delete[] ppNdbRecAttrDCnt;
ppNdbRecAttrDCnt = NULL ;
return NULL;
}
int runPkReadMultiBasic(NDBT_Context* ctx, NDBT_Step* step){
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
const int MAX_NDBS = 200;
Ndb* pNdb = GETNDB(step);
Ndb_cluster_connection* conn = &pNdb->get_ndb_cluster_connection();
int i = 0;
HugoOperations hugoOps(*ctx->getTab());
Ndb* ndbObjs[ MAX_NDBS ];
NdbTransaction* transArray[ MAX_NDBS ];
Ndb ** ready_ndbs;
for (int j=0; j < MAX_NDBS; j++)
{
Ndb* ndb = new Ndb(conn);
check(ndb->init() == 0, (*ndb));
ndbObjs[ j ] = ndb;
}
while (i<loops) {
ndbout << "Loop : " << i << ": ";
int recordsLeft = records;
do
{
/* Define and execute Pk read requests on
* different Ndb objects
*/
int ndbcnt = 0;
int pollcnt = 0;
int lumpsize = 1 + myRandom48(MIN(recordsLeft, MAX_NDBS));
while(lumpsize &&
recordsLeft &&
ndbcnt < MAX_NDBS)
{
Ndb* ndb = ndbObjs[ ndbcnt ];
NdbTransaction* trans = ndb->startTransaction();
check(trans != NULL, (*ndb));
NdbOperation* readOp = trans->getNdbOperation(ctx->getTab());
check(readOp != NULL, (*trans));
check(readOp->readTuple() == 0, (*readOp));
check(hugoOps.equalForRow(readOp, recordsLeft) == 0, hugoOps);
/* Read all other cols */
for (int k=0; k < ctx->getTab()->getNoOfColumns(); k++)
{
check(readOp->getValue(ctx->getTab()->getColumn(k)) != NULL,
(*readOp));
}
/* Now send em off */
trans->executeAsynchPrepare(NdbTransaction::Commit,
NULL,
NULL,
NdbOperation::AbortOnError);
ndb->sendPreparedTransactions();
transArray[ndbcnt] = trans;
global_poll_group->addNdb(ndb);
ndbcnt++;
pollcnt++;
recordsLeft--;
lumpsize--;
};
/* Ok, now wait for the Ndbs to complete */
while (pollcnt)
{
/* Occasionally check with no timeout */
Uint32 timeout_millis = myRandom48(2)?10000:0;
int count = global_poll_group->wait(ready_ndbs, timeout_millis);
if (count > 0)
{
for (int y=0; y < count; y++)
{
Ndb *ndb = ready_ndbs[y];
check(ndb->pollNdb(0, 1) != 0, (*ndb));
}
pollcnt -= count;
}
}
/* Ok, now close the transactions */
for (int t=0; t < ndbcnt; t++)
{
transArray[t]->close();
}
} while (recordsLeft);
i++;
}
for (int j=0; j < MAX_NDBS; j++)
{
delete ndbObjs[ j ];
}
return NDBT_OK;
}
int
NdbBackup::Fail(NdbRestarter& _restarter, int *Fail_codes, const int sz, bool onMaster){
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
int nNodes = _restarter.getNumDbNodes();
myRandom48Init(NdbTick_CurrentMillisecond());
for(int i = 0; i<sz; i++){
int error = Fail_codes[i];
unsigned int backupId;
const int masterNodeId = _restarter.getMasterNodeId();
CHECK(masterNodeId > 0, "getMasterNodeId failed");
int nodeId;
nodeId = masterNodeId;
if (!onMaster) {
int randomId;
while (nodeId == masterNodeId) {
randomId = myRandom48(nNodes);
nodeId = _restarter.getDbNodeId(randomId);
}
}
g_err << "NdbBackup::Fail node = " << nodeId
<< " error code = " << error << " masterNodeId = "
<< masterNodeId << endl;
CHECK(_restarter.insertErrorInNode(nodeId, error) == 0,
"failed to set error insert");
g_info << "error inserted" << endl;
g_info << "waiting some before starting backup" << endl;
g_info << "starting backup" << endl;
int r = start(backupId);
g_info << "r = " << r
<< " (which should fail) started with id = " << backupId << endl;
if (r == 0) {
g_err << "Backup should have failed on error_insertion " << error << endl
<< "Master = " << masterNodeId << "Node = " << nodeId << endl;
return NDBT_FAILED;
}
CHECK(_restarter.waitClusterStarted() == 0,
"waitClusterStarted failed");
CHECK(_restarter.insertErrorInNode(nodeId, 10099) == 0,
"failed to set error insert");
NdbSleep_SecSleep(5);
int val2[] = { 24, 2424 };
CHECK(_restarter.dumpStateAllNodes(val2, 2) == 0,
"failed to check backup resources RestartOnErrorInsert");
}
return NDBT_OK;
}
