void DatasetTableTailer::handleUpdate(NdbRecAttr* value[]) {
    int datasetPK = value[DS_ID_PK]->int32_value();
    int datasetId = -1;
    int projectId = -1;
    if(value[DS_INODE_ID]->isNULL() == -1){
        const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
        const NdbDictionary::Table* table = getTable(database, TABLE.mTableName);
        NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
        NdbOperation* op = getNdbOperation(transaction, table);
        
        op->readTuple(NdbOperation::LM_CommittedRead);
        
        op->equal(_dataset_cols[8].c_str(), datasetPK);
        
        NdbRecAttr* datasetIdCol = getNdbOperationValue(op, _dataset_cols[DS_INODE_ID]);
        NdbRecAttr* projectIdCol = getNdbOperationValue(op, _dataset_cols[DS_PROJ_ID]);
        
        executeTransaction(transaction, NdbTransaction::Commit);        
        datasetId = datasetIdCol->int32_value();
        projectId = projectIdCol->int32_value();
        
        transaction->close();
    }
    
    if(datasetId == -1 || projectId == -1){
        LOG_ERROR("Couldn't resolve projectId[" << projectId << "] or datasetId[" << datasetId << "]");
        return;
    }
    
     
    string data = createJSONUpSert(projectId, value);
    if (mElasticSearch->addDataset(projectId, datasetId, data)) {
        LOG_INFO("Update Dataset[" << datasetId << "]: Succeeded");
    }
}
void DatasetTableTailer::handleAdd(int datasetId, int projectId, NdbRecAttr* value[]) {
    mPDICache->addDatasetToProject(datasetId, projectId);
    
    string data = createJSONUpSert(projectId, value);
    if (mElasticSearch->addDataset(projectId, datasetId, data)) {
        LOG_INFO("Add Dataset[" << datasetId << "]: Succeeded");
    }

    const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
    NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
    Recovery::checkpointDataset(database, transaction, datasetId);
    transaction->close();
}
Beispiel #3
0
int runBuddyTransTimeout(NDBT_Context* ctx, NDBT_Step* step){
  int result = NDBT_OK;
  int loops = ctx->getNumLoops();
  int records = ctx->getNumRecords();
  int stepNo = step->getStepNo();
  ndbout << "TransactionInactiveTimeout="<< TIMEOUT <<endl;

  HugoOperations hugoOps(*ctx->getTab());
  Ndb* pNdb = GETNDB(step);

  for (int l = 1; l < loops && result == NDBT_OK; l++){

    NdbTransaction* pTrans = 0;
    do{
      pTrans = pNdb->startTransaction();
      NdbScanOperation* pOp = pTrans->getNdbScanOperation(ctx->getTab());
      CHECK(pOp->readTuples(NdbOperation::LM_Read, 0, 0, 1) == 0);
      CHECK(pTrans->execute(NoCommit) == 0);
      
      int sleep = 2 * TIMEOUT;
      ndbout << "Sleeping for " << sleep << " milliseconds" << endl;
      NdbSleep_MilliSleep(sleep);
    
      int res = 0;
      while((res = pOp->nextResult()) == 0);
      ndbout_c("res: %d", res);
      CHECK(res == -1);
      
    } while(false);
    
    if (pTrans)
    {
      pTrans->close();
    }
  }
  
  return result;
}
/* Consumer */
int runV2MultiWait_WaitPop_Thread(NDBT_Context* ctx, NDBT_Step* step)
{
  static int iter = 0;  // keeps incrementing when test case is repeated
  int records = ctx->getNumRecords();
  const char * d[5] = { " fast"," slow"," slow",""," slow" };
  const int timeout[3] = { 100, 1, 0 };
  const int pct_wait[9] = { 0,0,0,50,50,50,100,100,100 };
  for (int loop = 0; loop < V2_NLOOPS; loop++, iter++) 
  {
    ctx->incProperty("LOOP");
    ndbout << "V2 test: " << d[loop&1] << d[loop&2] << d[loop&4];
    ndbout << " " << timeout[iter%3] << "/" << pct_wait[iter%9] << endl;
    bool slow = loop & 4; 
    int nrec = 0;
    while(nrec < records) 
    {
      /* Occasionally check with no timeout */
      global_poll_group->wait(timeout[iter%3], pct_wait[iter%9]);
      Ndb * ndb = global_poll_group->pop();
      while(ndb) 
      {
        check(ndb->pollNdb(0, 1) != 0, (*ndb));
        nrec++;
        NdbTransaction *tx = (NdbTransaction *) ndb->getCustomData();
        tx->close();
        global_ndb_pool->recycleNdb(ndb);
        ndb = global_poll_group->pop();
      }
      if(slow) 
      {
         NdbSleep_MilliSleep(myRandom48(6));
      }  
    }
  }
  ctx->stopTest();
  global_ndb_pool->closeAll();
  return NDBT_OK;
}
Beispiel #5
0
void TableTailer::recover(int recoverFromId) {
    const NdbDictionary::Dictionary* database = getDatabase(mNdbConnection);
    const NdbDictionary::Index* index = getIndex(database, mTable.mTableName, mTable.mRecoveryIndex);
    
    NdbTransaction* transaction = startNdbTransaction(mNdbConnection);
    NdbIndexScanOperation* scanOp = getNdbIndexScanOperation(transaction, index);
    
    scanOp->readTuples(NdbOperation::LM_CommittedRead, NdbScanOperation::SF_OrderBy);
    scanOp->setBound(mTable.mRecoveryColumn.c_str(), NdbIndexScanOperation::BoundLT, (char*) & recoverFromId);
    
    NdbRecAttr * row[mTable.mNoColumns];
    
    for (int i = 0; i < mTable.mNoColumns; i++) {
        row[i] = scanOp->getValue(mTable.mColumnNames[i].c_str());
    }

    executeTransaction(transaction, NdbTransaction::Commit);
    
    while (scanOp->nextResult(true) == 0) {
        handleEvent(NdbDictionary::Event::TE_INSERT, NULL, row);
    }
    
    transaction->close();
}
Beispiel #6
0
int populate(Ndb * myNdb)
{
  int i;
  Car cars[15];

  const NdbDictionary::Dictionary* myDict= myNdb->getDictionary();
  const NdbDictionary::Table *myTable= myDict->getTable("api_scan");

  if (myTable == NULL) 
    APIERROR(myDict->getNdbError());

  /**
   * Five blue mercedes
   */
  for (i = 0; i < 5; i++)
  {
    cars[i].reg_no = i;
    sprintf(cars[i].brand, "Mercedes");
    sprintf(cars[i].color, "Blue");
  }

  /**
   * Five black bmw
   */
  for (i = 5; i < 10; i++)
  {
    cars[i].reg_no = i;
    sprintf(cars[i].brand, "BMW");
    sprintf(cars[i].color, "Black");
  }

  /**
   * Five pink toyotas
   */
  for (i = 10; i < 15; i++)
  {
    cars[i].reg_no = i;
    sprintf(cars[i].brand, "Toyota");
    sprintf(cars[i].color, "Pink");
  }
  
  NdbTransaction* myTrans = myNdb->startTransaction();
  if (myTrans == NULL)
    APIERROR(myNdb->getNdbError());

  for (i = 0; i < 15; i++) 
  {
    NdbOperation* myNdbOperation = myTrans->getNdbOperation(myTable);
    if (myNdbOperation == NULL) 
      APIERROR(myTrans->getNdbError());
    myNdbOperation->insertTuple();
    myNdbOperation->equal("REG_NO", cars[i].reg_no);
    myNdbOperation->setValue("BRAND", cars[i].brand);
    myNdbOperation->setValue("COLOR", cars[i].color);
  }

  int check = myTrans->execute(NdbTransaction::Commit);

  myTrans->close();

  return check != -1;
}
Beispiel #7
0
int calc_var_column(const NdbDictionary::Table * t,
		    const NdbDictionary::Index * ix,
		    int col,
		    Ndb* ndb,
		    bool longvarchar,
		    bool ftScan,
            bool ignoreData)
{
  
  char buff[8052];
  memset(buff, 0, sizeof(buff));
  int sz=0;
  NdbTransaction * trans = ndb->startTransaction();
  if(trans==0)
    abort();

  NdbScanOperation * sop;
  sop=trans->getNdbScanOperation(t);

  sop->readTuples();
  NdbRecAttr * attr=sop->getValue(col, buff);
  int rows=0;
  int charset_size=1;
  bool no_data=false;  

  //Set charset cost (for binary and latin1 cost is 1)
  const NdbDictionary::Column  * c = t->getColumn(col);
  const NdbDictionary::Column::Type type = c->getType();
  if ((type == NdbDictionary::Column::Varchar) || (type == NdbDictionary::Column::Longvarchar))
  {
    CHARSET_INFO * cs2;
    cs2= (CHARSET_INFO*)(c->getCharset());

    if(cs2!=0)
    {       
      if(strncmp(cs2->name, "utf8",4)==0)
      {
        charset_size=3;
        printf(  "---\tWARNING! cs2->name charset used, each character cost : %d bytes\n",charset_size);
      }

      if(strncmp(cs2->name, "ucs2",4)==0)
      {
        charset_size=2;
        printf(  "---\tWARNING! cs2->name charset used, each character cost : %d bytes\n",charset_size);
      }
    }
  }

  //Set var header size
  int headerSize=longvarchar ? 2 : 1;



  if(trans->execute(NdbTransaction::NoCommit, 
		    NdbOperation::AbortOnError, 1) == -1)
  {
	no_data=true;
	trans->close();
  }


  if(!no_data)
  {
    int check=0;
    while( ((check = sop->nextResult(true)) == 0) && !ignoreData)
	{  
	  rows++;
      if (verbose) 
      {
        printf("attribut %d size = %d \n",rows,attr->isNULL()==1 ? 0 : attr->get_size_in_bytes());  
      }


      if(attr->isNULL()==1)  
      {
        sz+=0; //for the time being until we know for sure..
      }
      else
      { //attr->get_size_in_bytes return lengh of attr including header attr cost = (length-header*charset)
        sz+=((attr->get_size_in_bytes() - headerSize)* charset_size);
      }
            
	  if(rows==1024 && !ftScan)
	    break;
	}
    trans->close();
  }

  if(rows==0)
  {
    sz = (int)(((float)(t->getColumn(col)->getSizeInBytes()))* (float)((float)loadfactor/100));
    printf("---\tWARNING! No reference data found for VAR*. Defaulting to max size (loadfactor=100 percent)..%d bytes \n",sz);
    printf("\tConsider loading database with average data for exact measurement\n");
      
    return sz+waste_sz(sz);
  }

  int tmpsz=(sz/rows)+headerSize;
  sz=tmpsz + waste_sz(tmpsz);
  if(ix)
    printf("---\t\tVAR* attribute is %d bytes averaged over %d rows\n", sz, rows);
  else
    printf("---\tVAR* attribute is %d bytes averaged over %d rows\n", sz, rows);
  return sz;
}
Beispiel #8
0
int calc_blob_column(const NdbDictionary::Table * t,
		   const NdbDictionary::Index * ix,
		   int col,
		   Ndb* ndb,
		   int & szRam,
		   int & szDisk,
		   bool ftScan)
{

  NdbTransaction * trans = ndb->startTransaction();
  NdbScanOperation * sop = trans->getNdbScanOperation(t);

  sop->readTuples();

  NdbBlob * blob = sop->getBlobHandle(col);  
  bool no_data=false;
  if(trans->execute(NdbTransaction::NoCommit, 
		    NdbOperation::AbortOnError, 1) == -1)
    {
      no_data=true;
      sop->close();
      trans->close();
    }
  
  unsigned long long len=0;


  int rows=0;
  int check=0;
  const NdbDictionary::Column *  c = t->getColumn(col);
  int part_size= c->getPartSize();
  if(!no_data)
    {
      while(((check = sop->nextResult(true)) == 0) && !ignoreData)
	{
	  int isnull;
	  rows++;
	  blob->getNull(isnull);
	  if(isnull)
	    len=0;
	  else	    
	    blob->getLength(len);
	  
	  /*	  
		  printf("blob is %llu\n", len);
		  if(len>256)
		  {
		  szRam+=(((len-256)/part_size) + 1)*part_size+256;
		  printf("len2=%llu, part-size=%d, len=%llu\n", (((len-256)/part_size) + 1)*part_size+256, part_size, len);
		  
		  }
		  else
	  */
	  
	  szRam+=(int)len;
	  if(rows==1000 && !ftScan)
	    break;
	}
      sop->close();
      trans->close();
    }
  if(rows==0)
    {
      if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
      {
          printf("---\tWARNING! No reference data found for BLOB/TEXT. "
             "Defaulting to 256 bytes DataMemory, %d bytes Diskspace! \n",(part_size<=256 ? 0:part_size));
          printf("\tConsider loading database with average data for exact"
             " measurement. \n");
          szRam=256;
          szDisk=(part_size<=256 ? 0:part_size);
          return 0;
      }
      else
      {
         printf("---\tWARNING! No reference data found for BLOB/TEXT. "
	        "Defaulting to %d bytes DataMemory ! \n", (part_size<=256 ? 256:part_size+256));
         printf("\tConsider loading database with average data for exact"
	        " measurement. \n");
         szRam=(part_size<=256 ? 256:part_size+256);
         szDisk=0;
         return 0;
      }
    }
  if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
    {
    int averageSz=szRam/rows;
    if((averageSz)>256)
	  {
      szRam=256;
	  szDisk=((averageSz-256)/part_size) *part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
	  }
      else
	  {
      szRam=256;
	  szDisk=0;
	  }
    }
  else
    {
      int averageSz=szRam/rows;
      szDisk=0;
      if((averageSz)<256)
        {
        szRam=256;
        }
        else
        {
        szRam=256 + ((averageSz-256)/part_size)*part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
        }
    }
   
  printf("---\tBLOB/TEXT attribute is %d bytes (RAM) and %d bytes (DISK)"
	 " averaged over %d rows\n", 	 
	 szRam, 
	 szDisk,
	 rows);

  return 0;
}