void DatasetTableTailer::handleUpdate(NdbRecAttr* value[]) {
int datasetPK = value[DS_ID_PK]->int32_value();
int datasetId = -1;
int projectId = -1;
if(value[DS_INODE_ID]->isNULL() == -1){
const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
const NdbDictionary::Table* table = getTable(database, TABLE.mTableName);
NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
NdbOperation* op = getNdbOperation(transaction, table);
op->readTuple(NdbOperation::LM_CommittedRead);
op->equal(_dataset_cols[8].c_str(), datasetPK);
NdbRecAttr* datasetIdCol = getNdbOperationValue(op, _dataset_cols[DS_INODE_ID]);
NdbRecAttr* projectIdCol = getNdbOperationValue(op, _dataset_cols[DS_PROJ_ID]);
executeTransaction(transaction, NdbTransaction::Commit);
datasetId = datasetIdCol->int32_value();
projectId = projectIdCol->int32_value();
transaction->close();
}
if(datasetId == -1 || projectId == -1){
LOG_ERROR("Couldn't resolve projectId[" << projectId << "] or datasetId[" << datasetId << "]");
return;
}
string data = createJSONUpSert(projectId, value);
if (mElasticSearch->addDataset(projectId, datasetId, data)) {
LOG_INFO("Update Dataset[" << datasetId << "]: Succeeded");
}
}
void DatasetTableTailer::handleAdd(int datasetId, int projectId, NdbRecAttr* value[]) {
mPDICache->addDatasetToProject(datasetId, projectId);
string data = createJSONUpSert(projectId, value);
if (mElasticSearch->addDataset(projectId, datasetId, data)) {
LOG_INFO("Add Dataset[" << datasetId << "]: Succeeded");
}
const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
Recovery::checkpointDataset(database, transaction, datasetId);
transaction->close();
}
int runBuddyTransTimeout(NDBT_Context* ctx, NDBT_Step* step){
int result = NDBT_OK;
int loops = ctx->getNumLoops();
int records = ctx->getNumRecords();
int stepNo = step->getStepNo();
ndbout << "TransactionInactiveTimeout="<< TIMEOUT <<endl;
HugoOperations hugoOps(*ctx->getTab());
Ndb* pNdb = GETNDB(step);
for (int l = 1; l < loops && result == NDBT_OK; l++){
NdbTransaction* pTrans = 0;
do{
pTrans = pNdb->startTransaction();
NdbScanOperation* pOp = pTrans->getNdbScanOperation(ctx->getTab());
CHECK(pOp->readTuples(NdbOperation::LM_Read, 0, 0, 1) == 0);
CHECK(pTrans->execute(NoCommit) == 0);
int sleep = 2 * TIMEOUT;
ndbout << "Sleeping for " << sleep << " milliseconds" << endl;
NdbSleep_MilliSleep(sleep);
int res = 0;
while((res = pOp->nextResult()) == 0);
ndbout_c("res: %d", res);
CHECK(res == -1);
} while(false);
if (pTrans)
{
pTrans->close();
}
}
return result;
}
/* Consumer */
int runV2MultiWait_WaitPop_Thread(NDBT_Context* ctx, NDBT_Step* step)
{
static int iter = 0; // keeps incrementing when test case is repeated
int records = ctx->getNumRecords();
const char * d[5] = { " fast"," slow"," slow",""," slow" };
const int timeout[3] = { 100, 1, 0 };
const int pct_wait[9] = { 0,0,0,50,50,50,100,100,100 };
for (int loop = 0; loop < V2_NLOOPS; loop++, iter++)
{
ctx->incProperty("LOOP");
ndbout << "V2 test: " << d[loop&1] << d[loop&2] << d[loop&4];
ndbout << " " << timeout[iter%3] << "/" << pct_wait[iter%9] << endl;
bool slow = loop & 4;
int nrec = 0;
while(nrec < records)
{
/* Occasionally check with no timeout */
global_poll_group->wait(timeout[iter%3], pct_wait[iter%9]);
Ndb * ndb = global_poll_group->pop();
while(ndb)
{
check(ndb->pollNdb(0, 1) != 0, (*ndb));
nrec++;
NdbTransaction *tx = (NdbTransaction *) ndb->getCustomData();
tx->close();
global_ndb_pool->recycleNdb(ndb);
ndb = global_poll_group->pop();
}
if(slow)
{
NdbSleep_MilliSleep(myRandom48(6));
}
}
}
ctx->stopTest();
global_ndb_pool->closeAll();
return NDBT_OK;
}
void TableTailer::recover(int recoverFromId) {
const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
const NdbDictionary::Index* index = getIndex(database, mTable.mTableName, mTable.mRecoveryIndex);
NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
NdbIndexScanOperation* scanOp = getNdbIndexScanOperation(transaction, index);
scanOp->readTuples(NdbOperation::LM_CommittedRead, NdbScanOperation::SF_OrderBy);
scanOp->setBound(mTable.mRecoveryColumn.c_str(), NdbIndexScanOperation::BoundLT, (char*) & recoverFromId);
NdbRecAttr * row[mTable.mNoColumns];
for (int i = 0; i < mTable.mNoColumns; i++) {
row[i] = scanOp->getValue(mTable.mColumnNames[i].c_str());
}
executeTransaction(transaction, NdbTransaction::Commit);
while (scanOp->nextResult(true) == 0) {
handleEvent(NdbDictionary::Event::TE_INSERT, NULL, row);
}
transaction->close();
}
int populate(Ndb * myNdb)
{
int i;
Car cars[15];
const NdbDictionary::Dictionary* myDict= myNdb->getDictionary();
const NdbDictionary::Table *myTable= myDict->getTable("api_scan");
if (myTable == NULL)
APIERROR(myDict->getNdbError());
/**
* Five blue mercedes
*/
for (i = 0; i < 5; i++)
{
cars[i].reg_no = i;
sprintf(cars[i].brand, "Mercedes");
sprintf(cars[i].color, "Blue");
}
/**
* Five black bmw
*/
for (i = 5; i < 10; i++)
{
cars[i].reg_no = i;
sprintf(cars[i].brand, "BMW");
sprintf(cars[i].color, "Black");
}
/**
* Five pink toyotas
*/
for (i = 10; i < 15; i++)
{
cars[i].reg_no = i;
sprintf(cars[i].brand, "Toyota");
sprintf(cars[i].color, "Pink");
}
NdbTransaction* myTrans = myNdb->startTransaction();
if (myTrans == NULL)
APIERROR(myNdb->getNdbError());
for (i = 0; i < 15; i++)
{
NdbOperation* myNdbOperation = myTrans->getNdbOperation(myTable);
if (myNdbOperation == NULL)
APIERROR(myTrans->getNdbError());
myNdbOperation->insertTuple();
myNdbOperation->equal("REG_NO", cars[i].reg_no);
myNdbOperation->setValue("BRAND", cars[i].brand);
myNdbOperation->setValue("COLOR", cars[i].color);
}
int check = myTrans->execute(NdbTransaction::Commit);
myTrans->close();
return check != -1;
}
int calc_var_column(const NdbDictionary::Table * t,
const NdbDictionary::Index * ix,
int col,
Ndb* ndb,
bool longvarchar,
bool ftScan,
bool ignoreData)
{
char buff[8052];
memset(buff, 0, sizeof(buff));
int sz=0;
NdbTransaction * trans = ndb->startTransaction();
if(trans==0)
abort();
NdbScanOperation * sop;
sop=trans->getNdbScanOperation(t);
sop->readTuples();
NdbRecAttr * attr=sop->getValue(col, buff);
int rows=0;
int charset_size=1;
bool no_data=false;
//Set charset cost (for binary and latin1 cost is 1)
const NdbDictionary::Column * c = t->getColumn(col);
const NdbDictionary::Column::Type type = c->getType();
if ((type == NdbDictionary::Column::Varchar) || (type == NdbDictionary::Column::Longvarchar))
{
CHARSET_INFO * cs2;
cs2= (CHARSET_INFO*)(c->getCharset());
if(cs2!=0)
{
if(strncmp(cs2->name, "utf8",4)==0)
{
charset_size=3;
printf( "---\tWARNING! cs2->name charset used, each character cost : %d bytes\n",charset_size);
}
if(strncmp(cs2->name, "ucs2",4)==0)
{
charset_size=2;
printf( "---\tWARNING! cs2->name charset used, each character cost : %d bytes\n",charset_size);
}
}
}
//Set var header size
int headerSize=longvarchar ? 2 : 1;
if(trans->execute(NdbTransaction::NoCommit,
NdbOperation::AbortOnError, 1) == -1)
{
no_data=true;
trans->close();
}
if(!no_data)
{
int check=0;
while( ((check = sop->nextResult(true)) == 0) && !ignoreData)
{
rows++;
if (verbose)
{
printf("attribut %d size = %d \n",rows,attr->isNULL()==1 ? 0 : attr->get_size_in_bytes());
}
if(attr->isNULL()==1)
{
sz+=0; //for the time being until we know for sure..
}
else
{ //attr->get_size_in_bytes return lengh of attr including header attr cost = (length-header*charset)
sz+=((attr->get_size_in_bytes() - headerSize)* charset_size);
}
if(rows==1024 && !ftScan)
break;
}
trans->close();
}
if(rows==0)
{
sz = (int)(((float)(t->getColumn(col)->getSizeInBytes()))* (float)((float)loadfactor/100));
printf("---\tWARNING! No reference data found for VAR*. Defaulting to max size (loadfactor=100 percent)..%d bytes \n",sz);
printf("\tConsider loading database with average data for exact measurement\n");
return sz+waste_sz(sz);
}
int tmpsz=(sz/rows)+headerSize;
sz=tmpsz + waste_sz(tmpsz);
if(ix)
printf("---\t\tVAR* attribute is %d bytes averaged over %d rows\n", sz, rows);
else
printf("---\tVAR* attribute is %d bytes averaged over %d rows\n", sz, rows);
return sz;
}
int calc_blob_column(const NdbDictionary::Table * t,
const NdbDictionary::Index * ix,
int col,
Ndb* ndb,
int & szRam,
int & szDisk,
bool ftScan)
{
NdbTransaction * trans = ndb->startTransaction();
NdbScanOperation * sop = trans->getNdbScanOperation(t);
sop->readTuples();
NdbBlob * blob = sop->getBlobHandle(col);
bool no_data=false;
if(trans->execute(NdbTransaction::NoCommit,
NdbOperation::AbortOnError, 1) == -1)
{
no_data=true;
sop->close();
trans->close();
}
unsigned long long len=0;
int rows=0;
int check=0;
const NdbDictionary::Column * c = t->getColumn(col);
int part_size= c->getPartSize();
if(!no_data)
{
while(((check = sop->nextResult(true)) == 0) && !ignoreData)
{
int isnull;
rows++;
blob->getNull(isnull);
if(isnull)
len=0;
else
blob->getLength(len);
/*
printf("blob is %llu\n", len);
if(len>256)
{
szRam+=(((len-256)/part_size) + 1)*part_size+256;
printf("len2=%llu, part-size=%d, len=%llu\n", (((len-256)/part_size) + 1)*part_size+256, part_size, len);
}
else
*/
szRam+=(int)len;
if(rows==1000 && !ftScan)
break;
}
sop->close();
trans->close();
}
if(rows==0)
{
if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
{
printf("---\tWARNING! No reference data found for BLOB/TEXT. "
"Defaulting to 256 bytes DataMemory, %d bytes Diskspace! \n",(part_size<=256 ? 0:part_size));
printf("\tConsider loading database with average data for exact"
" measurement. \n");
szRam=256;
szDisk=(part_size<=256 ? 0:part_size);
return 0;
}
else
{
printf("---\tWARNING! No reference data found for BLOB/TEXT. "
"Defaulting to %d bytes DataMemory ! \n", (part_size<=256 ? 256:part_size+256));
printf("\tConsider loading database with average data for exact"
" measurement. \n");
szRam=(part_size<=256 ? 256:part_size+256);
szDisk=0;
return 0;
}
}
if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
{
int averageSz=szRam/rows;
if((averageSz)>256)
{
szRam=256;
szDisk=((averageSz-256)/part_size) *part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
}
else
{
szRam=256;
szDisk=0;
}
}
else
{
int averageSz=szRam/rows;
szDisk=0;
if((averageSz)<256)
{
szRam=256;
}
else
{
szRam=256 + ((averageSz-256)/part_size)*part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
}
}
printf("---\tBLOB/TEXT attribute is %d bytes (RAM) and %d bytes (DISK)"
" averaged over %d rows\n",
szRam,
szDisk,
rows);
return 0;
}