int HugoOperations::setValueForAttr(NdbOperation* pOp,
int attrId,
int rowId,
int updateId){
const NdbDictionary::Column* attr = tab.getColumn(attrId);
if (! (attr->getType() == NdbDictionary::Column::Blob))
{
int len = attr->getSizeInBytes();
char buf[NDB_MAX_TUPLE_SIZE];
memset(buf, 0, sizeof(buf));
Uint32 real_len;
const char * value = calc.calcValue(rowId, attrId,
updateId, buf, len, &real_len);
return pOp->setValue( attr->getName(), value, real_len);
}
else
{
char buf[32000];
int len = (int)sizeof(buf);
Uint32 real_len;
const char * value = calc.calcValue(rowId, attrId,
updateId, buf, len, &real_len);
NdbBlob * b = pOp->getBlobHandle(attrId);
if (b == 0)
return -1;
if (real_len == 0)
return b->setNull();
else
return b->setValue(value, real_len);
}
}
NdbBlob*
Ndb::getNdbBlob()
{
NdbBlob* tBlob = theImpl->theNdbBlobIdleList.seize(this);
if(tBlob)
{
tBlob->init();
}
return tBlob;
}
/* set all the unique attrs of this objectclass into the table
*/
extern "C" int
ndb_oc_attrs(
NdbTransaction *txn,
const NdbDictionary::Table *myTable,
Entry *e,
NdbOcInfo *no,
NdbAttrInfo **attrs,
int nattrs,
Attribute *old
)
{
char buf[65538], *ptr;
Attribute **an, **ao, *a;
NdbOperation *myop;
int i, j, max = 0;
int changed, rc;
Uint64 eid = e->e_id;
if ( !nattrs )
return 0;
an = (Attribute **)ch_malloc( 2 * nattrs * sizeof(Attribute *));
ao = an + nattrs;
/* Turn lists of attrs into arrays for easier access */
for ( i=0; i<nattrs; i++ ) {
if ( attrs[i]->na_oi != no ) {
an[i] = NULL;
ao[i] = NULL;
continue;
}
for ( a=e->e_attrs; a; a=a->a_next ) {
if ( a->a_desc == slap_schema.si_ad_objectClass )
continue;
if ( a->a_desc->ad_type == attrs[i]->na_attr ) {
/* Don't process same attr twice */
if ( a->a_flags & SLAP_ATTR_IXADD )
a = NULL;
else
a->a_flags |= SLAP_ATTR_IXADD;
break;
}
}
an[i] = a;
if ( a && a->a_numvals > max )
max = a->a_numvals;
for ( a=old; a; a=a->a_next ) {
if ( a->a_desc == slap_schema.si_ad_objectClass )
continue;
if ( a->a_desc->ad_type == attrs[i]->na_attr )
break;
}
ao[i] = a;
if ( a && a->a_numvals > max )
max = a->a_numvals;
}
for ( i=0; i<max; i++ ) {
myop = NULL;
for ( j=0; j<nattrs; j++ ) {
if ( !an[j] && !ao[j] )
continue;
changed = 0;
if ( an[j] && an[j]->a_numvals > i ) {
/* both old and new are present, compare for changes */
if ( ao[j] && ao[j]->a_numvals > i ) {
if ( ber_bvcmp( &ao[j]->a_nvals[i], &an[j]->a_nvals[i] ))
changed = V_REP;
} else {
changed = V_INS;
}
} else {
if ( ao[j] && ao[j]->a_numvals > i )
changed = V_DEL;
}
if ( changed ) {
if ( !myop ) {
rc = LDAP_OTHER;
myop = txn->getNdbOperation( myTable );
if ( !myop ) {
goto done;
}
if ( old ) {
if ( myop->writeTuple()) {
goto done;
}
} else {
if ( myop->insertTuple()) {
goto done;
}
}
if ( myop->equal( EID_COLUMN, eid )) {
goto done;
}
if ( myop->equal( VID_COLUMN, i )) {
goto done;
}
}
if ( attrs[j]->na_flag & NDB_INFO_ATBLOB ) {
NdbBlob *myBlob = myop->getBlobHandle( attrs[j]->na_column );
rc = LDAP_OTHER;
if ( !myBlob ) {
Debug( LDAP_DEBUG_TRACE, "ndb_oc_attrs: getBlobHandle failed %s (%d)\n",
myop->getNdbError().message, myop->getNdbError().code, 0 );
goto done;
}
if ( slapMode & SLAP_TOOL_MODE )
ndb_flush_blobs = 1;
if ( changed & V_INS ) {
if ( myBlob->setValue( an[j]->a_vals[i].bv_val, an[j]->a_vals[i].bv_len )) {
Debug( LDAP_DEBUG_TRACE, "ndb_oc_attrs: blob->setValue failed %s (%d)\n",
myBlob->getNdbError().message, myBlob->getNdbError().code, 0 );
goto done;
}
} else {
if ( myBlob->setValue( NULL, 0 )) {
Debug( LDAP_DEBUG_TRACE, "ndb_oc_attrs: blob->setValue failed %s (%d)\n",
myBlob->getNdbError().message, myBlob->getNdbError().code, 0 );
goto done;
}
}
} else {
if ( changed & V_INS ) {
if ( an[j]->a_vals[i].bv_len > attrs[j]->na_len ) {
Debug( LDAP_DEBUG_ANY, "ndb_oc_attrs: attribute %s too long for column\n",
attrs[j]->na_name.bv_val, 0, 0 );
rc = LDAP_CONSTRAINT_VIOLATION;
goto done;
}
ptr = buf;
*ptr++ = an[j]->a_vals[i].bv_len & 0xff;
if ( attrs[j]->na_len > 255 ) {
/* MedVar */
*ptr++ = an[j]->a_vals[i].bv_len >> 8;
}
memcpy( ptr, an[j]->a_vals[i].bv_val, an[j]->a_vals[i].bv_len );
ptr = buf;
} else {
ptr = NULL;
}
if ( myop->setValue( attrs[j]->na_column, ptr )) {
rc = LDAP_OTHER;
goto done;
}
}
}
}
int calc_blob_column(const NdbDictionary::Table * t,
const NdbDictionary::Index * ix,
int col,
Ndb* ndb,
int & szRam,
int & szDisk,
bool ftScan)
{
NdbTransaction * trans = ndb->startTransaction();
NdbScanOperation * sop = trans->getNdbScanOperation(t);
sop->readTuples();
NdbBlob * blob = sop->getBlobHandle(col);
bool no_data=false;
if(trans->execute(NdbTransaction::NoCommit,
NdbOperation::AbortOnError, 1) == -1)
{
no_data=true;
sop->close();
trans->close();
}
unsigned long long len=0;
int rows=0;
int check=0;
const NdbDictionary::Column * c = t->getColumn(col);
int part_size= c->getPartSize();
if(!no_data)
{
while(((check = sop->nextResult(true)) == 0) && !ignoreData)
{
int isnull;
rows++;
blob->getNull(isnull);
if(isnull)
len=0;
else
blob->getLength(len);
/*
printf("blob is %llu\n", len);
if(len>256)
{
szRam+=(((len-256)/part_size) + 1)*part_size+256;
printf("len2=%llu, part-size=%d, len=%llu\n", (((len-256)/part_size) + 1)*part_size+256, part_size, len);
}
else
*/
szRam+=(int)len;
if(rows==1000 && !ftScan)
break;
}
sop->close();
trans->close();
}
if(rows==0)
{
if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
{
printf("---\tWARNING! No reference data found for BLOB/TEXT. "
"Defaulting to 256 bytes DataMemory, %d bytes Diskspace! \n",(part_size<=256 ? 0:part_size));
printf("\tConsider loading database with average data for exact"
" measurement. \n");
szRam=256;
szDisk=(part_size<=256 ? 0:part_size);
return 0;
}
else
{
printf("---\tWARNING! No reference data found for BLOB/TEXT. "
"Defaulting to %d bytes DataMemory ! \n", (part_size<=256 ? 256:part_size+256));
printf("\tConsider loading database with average data for exact"
" measurement. \n");
szRam=(part_size<=256 ? 256:part_size+256);
szDisk=0;
return 0;
}
}
if (c->getStorageType() == NdbDictionary::Column::StorageTypeDisk)
{
int averageSz=szRam/rows;
if((averageSz)>256)
{
szRam=256;
szDisk=((averageSz-256)/part_size) *part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
}
else
{
szRam=256;
szDisk=0;
}
}
else
{
int averageSz=szRam/rows;
szDisk=0;
if((averageSz)<256)
{
szRam=256;
}
else
{
szRam=256 + ((averageSz-256)/part_size)*part_size + (((averageSz-256)%part_size)==0 ? 0:part_size);
}
}
printf("---\tBLOB/TEXT attribute is %d bytes (RAM) and %d bytes (DISK)"
" averaged over %d rows\n",
szRam,
szDisk,
rows);
return 0;
}
int main(int argc, char** argv)
{
if (argc < 3)
{
std::cout << "Arguments are <connect_string cluster> <timeout> [m(merge events)|d(debug)].\n";
exit(-1);
}
const char *connectstring = argv[1];
int timeout = atoi(argv[2]);
ndb_init();
bool merge_events = argc > 3 && strchr(argv[3], 'm') != 0;
#ifdef VM_TRACE
bool dbug = argc > 3 && strchr(argv[3], 'd') != 0;
if (dbug) DBUG_PUSH("d:t:");
if (dbug) putenv("API_SIGNAL_LOG=-");
#endif
Ndb_cluster_connection *cluster_connection=
new Ndb_cluster_connection(connectstring); // Object representing the cluster
int r= cluster_connection->connect(5 /* retries */,
3 /* delay between retries */,
1 /* verbose */);
if (r > 0)
{
std::cout
<< "Cluster connect failed, possibly resolved with more retries.\n";
exit(-1);
}
else if (r < 0)
{
std::cout
<< "Cluster connect failed.\n";
exit(-1);
}
if (cluster_connection->wait_until_ready(30,30))
{
std::cout << "Cluster was not ready within 30 secs." << std::endl;
exit(-1);
}
Ndb* myNdb= new Ndb(cluster_connection,
"TEST_DB"); // Object representing the database
if (myNdb->init() == -1) APIERROR(myNdb->getNdbError());
const char *eventName= "CHNG_IN_t0";
const char *eventTableName= "t0";
const int noEventColumnName= 5;
const char *eventColumnName[noEventColumnName]=
{"c0",
"c1",
"c2",
"c3",
"c4"
};
// Create events
myCreateEvent(myNdb,
eventName,
eventTableName,
eventColumnName,
noEventColumnName,
merge_events);
// Normal values and blobs are unfortunately handled differently..
typedef union { NdbRecAttr* ra; NdbBlob* bh; } RA_BH;
int i, j, k, l;
j = 0;
while (j < timeout) {
// Start "transaction" for handling events
NdbEventOperation* op;
printf("create EventOperation\n");
if ((op = myNdb->createEventOperation(eventName)) == NULL)
APIERROR(myNdb->getNdbError());
op->mergeEvents(merge_events);
printf("get values\n");
RA_BH recAttr[noEventColumnName];
RA_BH recAttrPre[noEventColumnName];
// primary keys should always be a part of the result
for (i = 0; i < noEventColumnName; i++) {
if (i < 4) {
recAttr[i].ra = op->getValue(eventColumnName[i]);
recAttrPre[i].ra = op->getPreValue(eventColumnName[i]);
} else if (merge_events) {
recAttr[i].bh = op->getBlobHandle(eventColumnName[i]);
recAttrPre[i].bh = op->getPreBlobHandle(eventColumnName[i]);
}
}
// set up the callbacks
printf("execute\n");
// This starts changes to "start flowing"
if (op->execute())
APIERROR(op->getNdbError());
NdbEventOperation* the_op = op;
i= 0;
while (i < timeout) {
// printf("now waiting for event...\n");
int r = myNdb->pollEvents(1000); // wait for event or 1000 ms
if (r > 0) {
// printf("got data! %d\n", r);
while ((op= myNdb->nextEvent())) {
assert(the_op == op);
i++;
switch (op->getEventType()) {
case NdbDictionary::Event::TE_INSERT:
printf("%u INSERT", i);
break;
case NdbDictionary::Event::TE_DELETE:
printf("%u DELETE", i);
break;
case NdbDictionary::Event::TE_UPDATE:
printf("%u UPDATE", i);
break;
default:
abort(); // should not happen
}
printf(" gci=%d\n", (int)op->getGCI());
for (k = 0; k <= 1; k++) {
printf(k == 0 ? "post: " : "pre : ");
for (l = 0; l < noEventColumnName; l++) {
if (l < 4) {
NdbRecAttr* ra = k == 0 ? recAttr[l].ra : recAttrPre[l].ra;
if (ra->isNULL() >= 0) { // we have a value
if (ra->isNULL() == 0) { // we have a non-null value
if (l < 2)
printf("%-5u", ra->u_32_value());
else
printf("%-5.4s", ra->aRef());
} else
printf("%-5s", "NULL");
} else
printf("%-5s", "-"); // no value
} else if (merge_events) {
int isNull;
NdbBlob* bh = k == 0 ? recAttr[l].bh : recAttrPre[l].bh;
bh->getDefined(isNull);
if (isNull >= 0) { // we have a value
if (! isNull) { // we have a non-null value
Uint64 length = 0;
bh->getLength(length);
// read into buffer
unsigned char* buf = new unsigned char [length];
memset(buf, 'X', length);
Uint32 n = length;
bh->readData(buf, n); // n is in/out
assert(n == length);
// pretty-print
bool first = true;
Uint32 i = 0;
while (i < n) {
unsigned char c = buf[i++];
Uint32 m = 1;
while (i < n && buf[i] == c)
i++, m++;
if (! first)
printf("+");
printf("%u%c", m, c);
first = false;
}
printf("[%u]", n);
delete [] buf;
} else
printf("%-5s", "NULL");
} else
printf("%-5s", "-"); // no value
}
}
printf("\n");
}
}
} else
printf("timed out (%i)\n", timeout);
}
// don't want to listen to events anymore
if (myNdb->dropEventOperation(the_op)) APIERROR(myNdb->getNdbError());
the_op = 0;
j++;
}
{
NdbDictionary::Dictionary *myDict = myNdb->getDictionary();
if (!myDict) APIERROR(myNdb->getNdbError());
// remove event from database
if (myDict->dropEvent(eventName)) APIERROR(myDict->getNdbError());
}
delete myNdb;
delete cluster_connection;
ndb_end(0);
return 0;
}
int main(int argc, char* argv[])
{
// setlocale(LC_ALL, "");
if (argc < 2) {
printf("No connection string specified.\n");
print_help();
return -1;
}
if (argc < 3) {
printf("No database specified.\n");
print_help();
return -1;
}
if (argc < 4) {
printf("No filepath specified.\n");
print_help();
return -1;
}
if (argc < 5) {
printf("No table format file specified.\n");
print_help();
return -1;
}
if (argc >= 6) {
tNoOfParallelTrans = atoi(argv[5]);
}
if (argc >= 7) {
sleepTimeMilli = atoi(argv[6]);
}
strcpy(connstring, argv[1]);
strcpy(database, argv[2]);
strcpy(filepath, argv[3]);
strcpy(tablefilepath, argv[4]);
ifstream fin(tablefilepath);
// first line is table name.
string tmpTableName;
getline(fin, tmpTableName);
strcpy(tablename, tmpTableName.c_str());
// find a "nokey" in table name
istringstream lineReader(tmpTableName);
string firstpart;
if( getline(lineReader, firstpart, ' ') )
{
// printf("first part: %s", firstpart.c_str());
string secondpart;
if( getline(lineReader, secondpart) ) {
// "firstpart secondpart"
if (secondpart != NOKEY_IDENTIFIER) {
cerr << "ERROR: could not recognize identifier: " << secondpart << endl;
cerr << "Do you mean: '" << NOKEY_IDENTIFIER << "'?"<< endl;
exit(-1);
} else {
noKey = true;
}
// printf("Attr: %s %s\n", key.c_str(), value.c_str());
}
strcpy(tablename, firstpart.c_str());
}
// Initialize transaction array
for(int i = 0 ; i < MAXTRANS ; i++)
{
transaction[i].used = 0;
transaction[i].conn = 0;
}
// each line is a column
for (string row; getline(fin, row); ) {
istringstream lineReader(row);
string key;
if( getline(lineReader, key, ' ') )
{
string value;
if( getline(lineReader, value) ) {
// "key value"
fieldName.push_back(key);
fieldType.push_back(value);
// printf("Attr: %s %s\n", key.c_str(), value.c_str());
}
}
}
fin.close();
Ndb_cluster_connection *conn = connect_to_cluster(connstring);
Ndb* ndb = new Ndb(conn, database);
if (ndb->init(1024) == -1)
{
// pass
}
ndb->waitUntilReady(10000);
printf("Connected: database [%s], connstr [%s], #parallelTrans=[%d]. Load table [%s] from file [%s]...\n", database, connstring, tNoOfParallelTrans, tablename, filepath);
// do_insert(*ndb);
const NdbDictionary::Dictionary* myDict= ndb->getDictionary();
// printf("table name: %s\n", tablename);
const NdbDictionary::Table *myTable= myDict->getTable(tablename);
if (myTable == NULL)
APIERROR(myDict->getNdbError());
// Load the data
bool dataleft = false;
typedef vector<vector<string> > Rows;
// Rows rows;
ifstream input(filepath);
char const row_delim = '\n';
char const field_delim = '\t';
int rowCounter = 0;
for (string row; getline(input, row, row_delim); ) {
// Find a slot in the transaction array
async_callback_t * cb;
// int retries = 0;
int current = -1;
int cursor = transTail + 1;
if (cursor >= MAXTRANS) {
cursor = 0;
}
for(int retries = 0; retries < MAX_TRANSALLOC_RETRY; retries++) {
// for(int cursor=0; cursor < MAXTRANS; cursor++)
while(true)
{
if(transaction[cursor].used == 0)
{
current = cursor;
cb = new async_callback_t;
/**
* Set data used by the callback
*/
cb->ndb = ndb; //handle to Ndb object so that we can close transaction
// in the callback (alt. make myNdb global).
cb->transaction = current; //This is the number (id) of this transaction
transaction[current].used = 1 ; //Mark the transaction as used
transTail = current; // optimizing scan
break;
}
else { // used
cursor += 1;
if (cursor >= MAXTRANS) {
cursor = 0;
}
}
}
if(current == -1) {
cerr << "WARNING: Number of transactions in parallel exceeds the maximum. retrying..." << endl;
usleep(1000);
continue;
} else {
break;
}
}
transaction[current].conn = ndb->startTransaction();
istringstream ss(row);
// NdbOperation *myOperation= myTransaction->getNdbOperation(myTable);
NdbOperation *myOperation= transaction[current].conn->getNdbOperation(myTable);
myOperation->insertTuple();
// If no primary key is assigned, have to set the tupleId explicitly
if (noKey) {
unsigned long long tupleId = 0;
// int
// Ndb::getAutoIncrementValue(const char* aTableName,
// Uint64 & autoValue, Uint32 cacheSize,
// Uint64 step, Uint64 start)
if (ndb->getAutoIncrementValue(myTable, tupleId, TUPLEID_FETCH_SIZE, 1, 1) != 0) {
cerr << "Error occurs while getting tupleID to insert.\n";
exit(-1);
}
myOperation->equal("$PK", tupleId);
//if (tupleId % 10000 == 0) {
// cerr <<"DEBUG: set tupleID to " << tupleId << endl;
//}
}
// Iterate for each field
int i = 0;
for (string field; getline(ss, field, field_delim); i++) {
// For NULL value, do not set value for this field
if (strcmp(field.c_str(), "\\N") == 0) {
continue;
}
if (strcmp(fieldType[i].c_str(), "int") == 0) {
// using a int64 to prevent problems..
long long value = atoll(field.c_str());
myOperation->setValue(fieldName[i].c_str(), value);
}
if (strcmp(fieldType[i].c_str(), "real") == 0) {
double value = atof(field.c_str());
myOperation->setValue(fieldName[i].c_str(), value);
}
if (strcmp(fieldType[i].c_str(), "varchar") == 0) {
char buffer[65535] = {};
make_ndb_varchar(buffer, field.c_str());
myOperation->setValue(fieldName[i].c_str(), buffer);
}
if (strcmp(fieldType[i].c_str(), "char") == 0) {
char buffer[65535] = {};
make_ndb_char(buffer, field.c_str());
myOperation->setValue(fieldName[i].c_str(), buffer);
// myOperation->setValue(fieldName[i].c_str(), field.c_str());
}
if (strcmp(fieldType[i].c_str(), "boolean") == 0) {
int value = atoi(field.c_str());
myOperation->setValue(fieldName[i].c_str(), value);
}
if (strcmp(fieldType[i].c_str(), "text") == 0) {
NdbBlob *myBlobHandle = myOperation->getBlobHandle(fieldName[i].c_str());
if (myBlobHandle == NULL) {
cerr << "Hint: in the TSV file any TEXT/BLOB attribute must come after the primary key column.\n";
APIERROR(myOperation->getNdbError());
}
myBlobHandle->setValue(field.c_str(), field.length());
// myBlobHandle->setNull();
}
}
transaction[current].conn->executeAsynchPrepare( NdbTransaction::Commit,
&callback,
cb
);
nPreparedTransactions++;
rowCounter++;
dataleft = true;
/**
* When we have prepared parallelism number of transactions ->
* send the transaction to ndb.
* Next time we will deal with the transactions are in the
* callback. There we will see which ones that were successful
* and which ones to retry.
*/
if (nPreparedTransactions >= tNoOfParallelTrans)
{
// send-poll all transactions
// close transaction is done in callback
ndb->sendPollNdb(3000, tNoOfParallelTrans );
nPreparedTransactions=0;
dataleft = false;
usleep(sleepTimeMilli);
}
// The SYNC way that can set multiple operations in one commit:
// if (myTransaction->execute( NdbTransaction::NoCommit ) == -1)
// APIERROR(myTransaction->getNdbError());
// if (rowCounter % TRANACTION_SIZE == 0) {
// // commit
// if (myTransaction->execute( NdbTransaction::Commit ) == -1)
// APIERROR(myTransaction->getNdbError());
// ndb->closeTransaction(myTransaction);
// myTransaction = ndb->startTransaction();
// dataleft = false;
// }
}
if (dataleft) {
ndb->sendPollNdb(3000, nPreparedTransactions );
nPreparedTransactions=0;
// SYNC way
// if (myTransaction->execute( NdbTransaction::Commit ) == -1)
// APIERROR(myTransaction->getNdbError());
// ndb->closeTransaction(myTransaction);
}
ndb->waitUntilReady(10000);
delete ndb;
disconnect_from_cluster(conn);
return EXIT_SUCCESS;
}
