void db_multiple_key_next(void *pointer, DBT *data,
unsigned char **key, u_int32_t *ret_key_size,
unsigned char **result, u_int32_t *result_size) {
DB_MULTIPLE_KEY_NEXT(pointer, data,
*key, *ret_key_size,
*result, *result_size);
}
static void
b_inmem_op_ds_bulk(u_int ops, u_int *totalp)
{
DB_ENV *dbenv;
DB *dbp;
DBC *dbc;
DBT key, data;
u_int32_t len, klen;
u_int i, total;
char *keybuf, *databuf;
void *pointer, *dp, *kp;
DB_MPOOL_STAT *gsp;
DB_BENCH_ASSERT((keybuf = malloc(keysize)) != NULL);
DB_BENCH_ASSERT((databuf = malloc(bulkbufsize)) != NULL);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
key.data = keybuf;
key.size = keysize;
data.data = databuf;
data.size = datasize;
memset(databuf, 'b', datasize);
DB_BENCH_ASSERT(db_create(&dbp, NULL, 0) == 0);
dbenv = dbp->dbenv;
dbp->set_errfile(dbp, stderr);
DB_BENCH_ASSERT(dbp->set_pagesize(dbp, pagesize) == 0);
DB_BENCH_ASSERT(dbp->set_cachesize(dbp, 0, cachesize, 1) == 0);
DB_BENCH_ASSERT(
dbp->open(dbp, NULL, NULL, NULL, DB_BTREE, DB_CREATE, 0666) == 0);
for (i = 1; i <= numitems; ++i) {
(void)snprintf(keybuf, keysize, "%7d", i);
DB_BENCH_ASSERT(dbp->put(dbp, NULL, &key, &data, 0) == 0);
}
#if 0
fp = fopen("before", "w");
dbp->set_msgfile(dbp, fp);
DB_BENCH_ASSERT (dbp->stat_print(dbp, DB_STAT_ALL) == 0);
#endif
DB_BENCH_ASSERT(dbp->cursor(dbp, NULL, &dbc, 0) == 0);
data.ulen = bulkbufsize;
data.flags = DB_DBT_USERMEM;
(void)dbenv->memp_stat(dbenv, &gsp, NULL, DB_STAT_CLEAR);
TIMER_START;
for (total = 0; ops > 0; --ops) {
DB_BENCH_ASSERT(dbc->c_get(
dbc, &key, &data, DB_FIRST | DB_MULTIPLE_KEY) == 0);
DB_MULTIPLE_INIT(pointer, &data);
while (pointer != NULL) {
DB_MULTIPLE_KEY_NEXT(pointer, &data, kp, klen, dp, len);
if (kp != NULL)
++total;
}
}
TIMER_STOP;
*totalp = total;
if (dbenv->memp_stat(dbenv, &gsp, NULL, 0) == 0)
DB_BENCH_ASSERT(gsp->st_cache_miss == 0);
#if 0
fp = fopen("before", "w");
dbp->set_msgfile(dbp, fp);
DB_BENCH_ASSERT (dbp->stat_print(dbp, DB_STAT_ALL) == 0);
#endif
DB_BENCH_ASSERT(dbp->close(dbp, 0) == 0);
COMPQUIET(dp, NULL);
COMPQUIET(klen, 0);
COMPQUIET(len, 0);
}
bool BulkHeaderGroup::BulkHeaderGroupBody()
{
// belt and braces
key.set_data(keymem);
data.set_data(datamem);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkHeaderGroup, JobList::Running, job->seq);
NewsGroup* ng = job->ng;
Db* db = ng->getDb();
MultiPartHeader mph;
SinglePartHeader sph;
HeaderBase* hb = 0;
HeaderGroup* headerGroup = 0;
HeaderGroup* advancedHeaderGroup = 0;
// typedef QMap<QString, QString> HeaderGroupIndexes; // subj, headerGroup index
// typedef QMap<QString, HeaderGroup*> HeaderGroups; // headerGroup index, headerGroup *
HeaderGroupIndexes headerGroupIndexes;
HeaderGroups headerGroups;
DBC *dbcp = 0;
DBT ckey, cdata;
memset(&ckey, 0, sizeof(ckey));
memset(&cdata, 0, sizeof(cdata));
size_t retklen, retdlen;
void *retkey = 0, *retdata = 0;
int ret, t_ret;
void *p = 0;
quint64 count=0;
cdata.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
cdata.flags = DB_DBT_USERMEM;
ckey.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
ckey.flags = DB_DBT_USERMEM;
/* Acquire a cursor for the database. */
if ((ret = db->get_DB()->cursor(db->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
{
db->err(ret, "DB->cursor");
char* ptr = 0;
ptr = (char*)(ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = (char*)(cdata.data);
Q_DELETE_ARRAY(ptr);
return false;
}
// To save the group records
ng->articlesNeedDeleting(false);
// Store the data in the database - flush first ...
u_int32_t delCount;
uchar keymem[KEYMEM_SIZE];
uchar datamem[DATAMEM_SIZE];
Dbt key, data;
char* p2 = 0;
QByteArray ba;
const char *k = 0;
key.set_flags(DB_DBT_USERMEM);
key.set_data(&keymem);
key.set_ulen(KEYMEM_SIZE);
data.set_flags(DB_DBT_USERMEM);
data.set_ulen(DATAMEM_SIZE);
data.set_data(&datamem);
QString subj = "MDQuban", from = "MDQuban";
//QString rs1 = "^(.*)(\".*\")";
//QString rs2 = "^(.*)\\s-\\s(.*)$";
//QString rs3 = "^(\\S+.*)\\[.*\\].*(\".*\")";
//QString rs3 = "^(.*)\\s-\\s.*\\s-\\s(.*)$";
QRegExp rx[3];
bool rxPosBack[3];
bool noRegexpGrouping;
QString recKey, storeIndex;
QString prevSubj = "MDQuban", prevFrom = "MDQuban";
int pos;
bool newGroup = false;
bool mphFound = false;
quint32 grouped = 0,
single = 0,
numGroups = 0;
qint16 stringDiff = -1;
bool prevGroup = false;
bool advancedPlacement = false;
bool skipAdvanced = false;
noRegexpGrouping = ng->isThereNoRegexOnGrouping();
if (noRegexpGrouping == false) // need regex for grouping
{
rx[0].setPattern(ng->getGroupRE1());
rx[1].setPattern(ng->getGroupRE2());
rx[2].setPattern(ng->getGroupRE3());
rxPosBack[0] = ng->getGroupRE1Back();
rxPosBack[1] = ng->getGroupRE2Back();
rxPosBack[2] = ng->getGroupRE3Back();
}
ng->getGroupingDb()->truncate(0, &delCount, 0);
qDebug() << "Deleted " << delCount << " records from group db";
QMapIterator<QString, QString> it(headerGroupIndexes);
QString advancedIndex;
for (;;)
{
/*
* Acquire the next set of key/data pairs. This code
* does not handle single key/data pairs that won't fit
* in a BUFFER_LENGTH size buffer, instead returning
* DB_BUFFER_SMALL to our caller.
*/
if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
{
if (ret != DB_NOTFOUND)
db->err(ret, "DBcursor->get");
break;
}
for (DB_MULTIPLE_INIT(p, &cdata);;)
{
DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
if (retdlen){;} // MD TODO compiler .... unused variable
recKey = QString::fromLocal8Bit((char*)retkey, retklen);
if (*((char *)retdata) == 'm')
{
MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
hb = (HeaderBase*)&mph;
mphFound = true;
}
else if (*((char *)retdata) == 's')
{
SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
hb = (HeaderBase*)&sph;
mphFound = false;
}
else
{
// What have we found ?????
qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
continue;
}
++count;
prevSubj = subj;
prevFrom = from;
subj = hb->getSubj();
from = hb->getFrom();
if (noRegexpGrouping == false) // need regex for grouping
{
for (int i=0; i<3; ++i)
{
if (rx[i].isEmpty() == false)
{
if (rxPosBack[i] == true) // from the back
{
pos = subj.lastIndexOf(rx[i]);
if (pos != -1)
subj.truncate(pos);
}
else // from the front
{
pos = rx[i].indexIn(subj);
if (pos > -1)
subj = rx[i].cap(0);
}
}
}
}
//qDebug() << "Stripped down to: " << subj;
stringDiff = -1;
if (prevFrom != from) // change of contributor
{
newGroup = true;
}
else // same contributor
{
if ((stringDiff = levenshteinDistance(prevSubj, subj)) > ng->getMatchDistance()) // no match ...
newGroup = true;
else
newGroup = false;
//qDebug() << "Diff between " << prevSubj << " and " << subj << " is " << stringDiff;
}
if (newGroup)
{
if (ng->isThereAdvancedGrouping())
{
it.toFront();
// decide if we can match to a previous group
while (it.hasNext())
{
it.next();
if ((stringDiff = levenshteinDistance(it.key(), subj)) <= ng->getMatchDistance()) // match ...
{
// The index for this group is in it.value()
// See if we have the HeaderGroup in our cache headerGroups)
if (headerGroups.contains(it.value()))
{
advancedHeaderGroup = headerGroups.value(it.value());
}
else // not in cache
{
advancedIndex = it.value();
advancedHeaderGroup = getGroup(ng, advancedIndex);
if (advancedHeaderGroup)
{
headerGroups.insert(advancedIndex, advancedHeaderGroup);
}
else // db read failed ..
{
skipAdvanced = true;
}
}
if (skipAdvanced == false)
{
if (mphFound)
advancedHeaderGroup->addMphKey(recKey);
else
advancedHeaderGroup->addSphKey(recKey);
advancedPlacement = true;
subj = prevSubj; // ignore this header as it's been placed out of sequence
from = prevFrom;
newGroup = false; // as we managed to relocate to an existing group
break; // stop looking at previous groups
}
else
skipAdvanced = false;
}
}
}
}
if (newGroup)
{
if (prevGroup) // save before moving on
{
ba = storeIndex.toLocal8Bit();
k= ba.constData();
memcpy(keymem, k, storeIndex.length());
key.set_size(storeIndex.length());
p2=headerGroup->data();
data.set_data(p2);
data.set_size(headerGroup->getRecordSize());
ret=ng->getGroupingDb()->put(NULL, &key, &data, 0);
if (ret!=0)
qDebug("Error updating record: %d", ret);
if (ng->isThereAdvancedGrouping())
headerGroupIndexes.insert(storeIndex.section('\n', 0, 0), storeIndex);
Q_DELETE_ARRAY(p2);
Q_DELETE(headerGroup);
numGroups++;
}
prevGroup = true;
storeIndex = subj % "\n" % from;
headerGroup = new HeaderGroup();
headerGroup->setDisplayName(subj);
headerGroup->setPostingDate(hb->getPostingDate());
headerGroup->setDownloadDate(hb->getDownloadDate());
headerGroup->setStatus(hb->getStatus());
headerGroup->setNextDistance(stringDiff);
}
// if we've found somewhere else to place this header then don't add again
if (!advancedPlacement)
{
if (mphFound)
headerGroup->addMphKey(recKey);
else
headerGroup->addSphKey(recKey);
}
else
advancedPlacement = false;
if (count % 250 == 0)
{
QCoreApplication::processEvents();
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if ((t_ret = dbcp->close(dbcp)) != 0)
{
db->err(ret, "DBcursor->close");
if (ret == 0)
ret = t_ret;
}
char* ptr = ((char*)ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = ((char*)cdata.data);
Q_DELETE_ARRAY(ptr);
if (headerGroups.count())
{
qDeleteAll(headerGroups);
headerGroups.clear();
}
qDebug() << "Multi = " << grouped << ", single = " << single;
ng->setHeadersNeedGrouping(false);
// Finally update the newsgroup
emit saveGroup(ng);
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit logEvent(tr("Bulk grouping of ") + ng->getTotal() + tr(" articles completed successfully."));
emit updateJob(JobList::BulkHeaderGroup, JobList::Finished_Ok, job->seq);
ng->setTotalGroups(numGroups);
Q_DELETE(headerGroup);
return true;
}
/*
* Use Bulk Get/Put to copy the given number of pages worth of
* records from the source database to the destination database,
* this function should be called until all tables are copied, at
* which point it will return SQLITE_DONE. Both Btrees need to
* have transactions before calling this function.
* p->pSrc - Source Btree
* p->tables - Contains a list of iTables to copy, gotten using
* btreeGetTables().
* p->currentTable - Index in tables of the current table being copied.
* p->srcCur - Cursor on the current source table being copied.
* p->pDest - Destiniation Btree.
* p->destCur - BtCursor on the destination table being copied into.
* pages - Number of pages worth of data to copy.
*/
static int btreeCopyPages(sqlite3_backup *p, int *pages)
{
DB *dbp;
DBT dataOut, dataIn;
char bufOut[MULTI_BUFSIZE], bufIn[MULTI_BUFSIZE];
int ret, rc, copied, srcIsDupIndex;
void *in, *out, *app;
ret = 0;
rc = SQLITE_OK;
dbp = NULL;
copied = 0;
memset(&dataOut, 0, sizeof(dataOut));
memset(&dataIn, 0, sizeof(dataIn));
dataOut.flags = DB_DBT_USERMEM;
dataIn.flags = DB_DBT_USERMEM;
dataOut.data = bufOut;
dataOut.ulen = sizeof(bufOut);
dataIn.data = bufIn;
dataIn.ulen = sizeof(bufIn);
while (*pages < 0 || *pages > copied) {
/* No tables left to copy */
if (p->tables[p->currentTable] == -1) {
u32 val;
/*
* Update the schema file format and largest rootpage
* in the meta data. Other meta data values should
* not be changed.
*/
sqlite3BtreeGetMeta(p->pSrc, 1, &val);
if (p->pSrc->db->errCode == SQLITE_BUSY) {
rc = SQLITE_BUSY;
goto err;
}
rc = sqlite3BtreeUpdateMeta(p->pDest, 1, val);
if (rc != SQLITE_OK)
goto err;
sqlite3BtreeGetMeta(p->pSrc, 3, &val);
if (p->pSrc->db->errCode == SQLITE_BUSY) {
rc = SQLITE_BUSY;
goto err;
}
rc = sqlite3BtreeUpdateMeta(p->pDest, 3, val);
if (rc != SQLITE_OK)
goto err;
ret = SQLITE_DONE;
goto err;
}
/* If not currently copying a table, get the next table. */
if (!p->srcCur) {
rc = btreeGetUserTable(p->pSrc, p->srcTxn, &dbp,
p->tables[p->currentTable]);
if (rc != SQLITE_OK)
goto err;
assert(dbp);
memset(&p->destCur, 0, sizeof(p->destCur));
/*
* Open a cursor on the destination table, this will
* create the table and allow the Btree to manage the
* DB object.
*/
sqlite3BtreeCursor(p->pDest, p->tables[p->currentTable],
1, dbp->app_private, &p->destCur);
if ((rc = p->destCur.error) != SQLITE_OK) {
app = dbp->app_private;
dbp->close(dbp, DB_NOSYNC);
if (app)
sqlite3DbFree(p->pSrcDb, app);
goto err;
}
/* Open a cursor on the source table. */
if ((ret = dbp->cursor(dbp,
p->srcTxn, &p->srcCur, 0)) != 0)
goto err;
dbp = 0;
}
srcIsDupIndex = isDupIndex((p->tables[p->currentTable] & 1) ?
BTREE_INTKEY : 0, p->pSrc->pBt->dbStorage,
p->srcCur->dbp->app_private, p->srcCur->dbp);
/*
* Copy the current table until the given number of
* pages is copied, or the entire table has been copied.
*/
while (*pages < 0 || *pages > copied) {
DBT key, data;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
/* Do a Bulk Get from the source table. */
ret = p->srcCur->get(p->srcCur, &key, &dataOut,
DB_NEXT | DB_MULTIPLE_KEY);
if (ret == DB_NOTFOUND)
break;
if (ret != 0)
goto err;
/* Copy the records into the Bulk buffer. */
DB_MULTIPLE_INIT(out, &dataOut);
DB_MULTIPLE_WRITE_INIT(in, &dataIn);
DB_MULTIPLE_KEY_NEXT(out, &dataOut, key.data,
key.size, data.data, data.size);
while (out) {
/*
* Have to translate the index formats if they
* are not the same.
*/
if (p->destCur.isDupIndex != srcIsDupIndex) {
if (srcIsDupIndex) {
p->destCur.key = key;
p->destCur.data = data;
if (!btreeCreateIndexKey(
&p->destCur)) {
rc = SQLITE_NOMEM;
goto err;
}
DB_MULTIPLE_KEY_WRITE_NEXT(in,
&dataIn,
p->destCur.index.data,
p->destCur.index.size,
p->destCur.data.data, 0);
} else {
/* Copy the key into the cursor
* index since spliting the key
* requires changing the
* internal memory.
*/
if (!allocateCursorIndex(
&p->destCur, key.size)) {
rc = SQLITE_NOMEM;
goto err;
}
memcpy(p->destCur.index.data,
key.data, key.size);
p->destCur.index.size =
key.size;
p->destCur.key.data =
p->destCur.index.data;
p->destCur.key.size =
p->destCur.index.size;
splitIndexKey(&p->destCur);
DB_MULTIPLE_KEY_WRITE_NEXT(
in, &dataIn,
p->destCur.key.data,
p->destCur.key.size,
p->destCur.data.data,
p->destCur.data.size);
}
} else
DB_MULTIPLE_KEY_WRITE_NEXT(in, &dataIn,
key.data, key.size,
data.data, data.size);
DB_MULTIPLE_KEY_NEXT(out, &dataOut,
key.data, key.size,
data.data, data.size);
}
/* Insert into the destination table. */
dbp = p->destCur.cached_db->dbp;
if ((ret = dbp->put(dbp, p->pDest->savepoint_txn,
&dataIn, 0, DB_MULTIPLE_KEY)) != 0)
goto err;
dbp = NULL;
copied += MULTI_BUFSIZE/SQLITE_DEFAULT_PAGE_SIZE;
}
/*
* Done copying the current table, time to look for a new
* table to copy.
*/
if (ret == DB_NOTFOUND) {
ret = 0;
rc = sqlite3BtreeCloseCursor(&p->destCur);
if (p->srcCur) {
app = p->srcCur->dbp->app_private;
dbp = p->srcCur->dbp;
p->srcCur->close(p->srcCur);
ret = dbp->close(dbp, DB_NOSYNC);
if (app)
sqlite3DbFree(p->pSrcDb, app);
}
p->srcCur = NULL;
if (ret != 0 || rc != SQLITE_OK)
goto err;
p->currentTable += 1;
}
}
goto done;
err: if (ret == SQLITE_DONE)
return ret;
done: return MAP_ERR(rc, ret);
}
int* ROSWOSSegment::lookupByIndex( char* key, int size, Dbc *iter,
u_int32_t flags, void **_ptr, void **buf )
{
size_t retklen, retdlen;
Dbt skey;
void *retkey, *retdata;
bool second_pass = false;
int answer;
memset(&skey, 0, sizeof(skey));
memset(&_bulk_data, 0, sizeof(_bulk_data));
//memset( (*buf), 0, sizeof( (*buf) ) );
//_bulk_data.set_data( _bulk_buffer );
_bulk_data.set_data( (*buf) );
_bulk_data.set_ulen( BUFFER_LENGTH );
_bulk_data.set_flags( DB_DBT_USERMEM );
skey.set_data( key );
skey.set_size( size );
//if ( _debug ) return NULL;
do
{
// if bulk buffer is done, get a new one.
//if ( _p_bulk == NULL )
if ( ((*_ptr)) == NULL )
{
//cout << " Request from iter " << iter << " key " << *(int *)key << " FLAGS? " << flags << endl;
memset( (*buf), 0, sizeof( (*buf) ) );
answer = iter->/*_sort_key_cursor->*/get( &skey, &_bulk_data,
flags | DB_MULTIPLE_KEY );
if ( answer == DB_NOTFOUND )
return NULL;
//DB_MULTIPLE_INIT(_p_bulk, _bulk_data.get_DBT());
DB_MULTIPLE_INIT((*_ptr), _bulk_data.get_DBT());
//cout << " DB_NOTFOUND " << DB_NOTFOUND << " ans " << answer << " -- Getting answers, page size " << BUFFER_LENGTH << " ITER " << iter << " PTR " << (*_ptr) <<endl;
}
//else
//cout << " ROSWOSSegment, *_ptr is not NULL, go go go" << endl;
/*if ( answer == DB_NOTFOUND )
{
//_debug = true;
// THAT really does NOT close the iterator. WHY, I have not
// a faintest idea!
//cout << " ITERATOR " << iter << " IS CLOSED " << endl;
//iter->close();
return NULL;
}*/
//if ( !_p_bulk )
if ( !((*_ptr)) )
return NULL;
// I don't understand that.
//if ( !_p_bulk && second_pass )
//cout << " PTR " << (*_ptr) << " SECOND PASS " << second_pass << endl;
if ( !((*_ptr)) && second_pass )
return NULL;
DB_MULTIPLE_KEY_NEXT((*_ptr), _bulk_data.get_DBT(), retkey, retklen, retdata, retdlen);
//DB_MULTIPLE_KEY_NEXT(_p_bulk, _bulk_data.get_DBT(), retkey, retklen, retdata, retdlen);
// cout << " ANd now, ptr is " << *_ptr << " int ptr " << ((int*)retkey) << endl;
second_pass = true;
}
//while( !_p_bulk ); // if bulk is null, get the buffer.
while( !(*_ptr) ); // if bulk is null, get the buffer.
/*if ( (*((int*)retkey)) > 10500 )
{
cout << DB_NOTFOUND << " <=> " << answer << " Ret answer " << *((int *)retkey) << " retDATA " << *((int *)retdata) << endl;
cout << " Recieved buffer " << buf << " points to buf " << (*buf) << " and ptr " << *_ptr << endl;
*/
/*if ( *((int *)retdata) == 596992 &&
10511 == *((int *)retkey) )
*(int *)retdata = 597221;
if ( *((int *)retdata) == 598272 &&
10524 == *((int *)retkey) )
*(int *)retdata = 598362;*/
//cout << " Finally returning " << ( ((int *)retdata) ) << endl << endl;
return( ((int *)retdata) );
/*
// THIS IS AN ITERATION SEQUENCE. DO NOT ERASE.
while ( _sort_key_cursor->get( &skey, &data, DB_NEXT | DB_MULTIPLE_KEY ) == 0 )
{
for (DB_MULTIPLE_INIT(p, data.get_DBT());;)
{
DB_MULTIPLE_KEY_NEXT(p, data.get_DBT(), retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
//cout << " Ret klen " << *((int *)retkey) << " retdlen " << *((int *)retdata) << endl;
//printf("key: %.*s, data: %.*s\n",
//(int)retklen, (char*)retkey, (int)retdlen, (char*)retdata);
ctr2++;
}
cout << " Got SORT key " << *((int *)skey.get_data()) << endl;
cout << " Curr count " << ctr2 << endl;
}
*/
/*
Dbt dkey, data; // Dbt is a key / data pair
int ret; //,a, b;
char *result = ( char* )malloc( sizeof(int) );
memset(&dkey, 0, sizeof(dkey));
memset(&data, 0, sizeof(data));
data.set_data( &result );
data.set_size( sizeof(int) );
// BDB documentation neglects to mention that once you use your own
// buffer you MUST set this parameter or things will break horribly.
// thanks to a helpful post on a google newsgroup...
data.set_ulen( sizeof(int) );
dkey.set_data( key );
dkey.set_size( size );
try
{
int before = StopWatch::ticks();
ret = iter->get( &dkey, &data, flags );
int after = StopWatch::ticks();
db_access+=(after-before)*StopWatch::secondsPerTick();
//cout << _segNameDebug << " Got " << (after-before)*StopWatch::secondsPerTick() << " for " << db_access << endl;
}
catch(DbException& e)
{
if (DEBUG) cout << "Exception " << e.what() << endl;
// Throw exception here
exit( 1 );
}
if ( ret < 0 )
{
//cout << ret << "Nothing found for key ... " << key << endl;
return NULL;
}
//int a,b;
//memcpy( &a, key, 4 );
//memcpy( &b, data.get_data(), 4 );
//cout << "Receiving key " << b << " for given key " << a << endl;
//if ( fake_key > 599874 )
//return NULL;
//memcpy( result, &fake_key, sizeof(int) );
//fake_key++;
memcpy( result, data.get_data(), sizeof(int) );
return ( int* )result;
*/
}
bool BulkLoad::bulkLoadBody()
{
if (m_cancel)
{
emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkLoad, JobList::Running, job->seq);
NewsGroup* ng = job->ng;
HeaderList* hl = job->headerList;
mphDeletionsList = 0;
sphDeletionsList = 0;
DBC *dbcp = 0;
MultiPartHeader mph;
SinglePartHeader sph;
HeaderBase* hb = 0;
memset(&ckey, 0, sizeof(ckey));
memset(&cdata, 0, sizeof(cdata));
cdata.data = (void *) dataBuffer;
cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
cdata.flags = DB_DBT_USERMEM;
ckey.data = (void *) keyBuffer;
ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
ckey.flags = DB_DBT_USERMEM;
size_t retklen, retdlen;
void *retkey = 0, *retdata = 0;
int ret, t_ret;
void *p = 0;
quint64 count=0;
QTime start = QTime::currentTime();
qDebug() << "Loading started: " << start.toString();
/* Acquire a cursor for the database. */
if ((ret = ng->getDb()->get_DB()->cursor(ng->getDb()->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
{
ng->getDb()->err(ret, "DB->cursor");
return false;
}
quint32 numIgnored = 0;
bool mphFound = false;
for (;;)
{
/*
* Acquire the next set of key/data pairs. This code
* does not handle single key/data pairs that won't fit
* in a BUFFER_LENGTH size buffer, instead returning
* DB_BUFFER_SMALL to our caller.
*/
if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
{
if (ret != DB_NOTFOUND)
ng->getDb()->err(ret, "DBcursor->get");
break;
}
for (DB_MULTIPLE_INIT(p, &cdata);;)
{
DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
if (retdlen){;} // MD TODO compiler .... unused variable
if (*((char *)retdata) == 'm')
{
MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
hb = (HeaderBase*)&mph;
mphFound = true;
}
else if (*((char *)retdata) == 's')
{
SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
hb = (HeaderBase*)&sph;
mphFound = false;
// qDebug() << "Single index = " << sph.getIndex();
}
else
{
// What have we found ?????
qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
continue;
}
if (hb->getStatus() & HeaderBase::MarkedForDeletion)
{
// ignore this header, garbage collection will remove it ...
++numIgnored;
++count; // ... but still count it as bulk delete will reduce the count
if (numIgnored == 1)
{
// These two will be freed by bulk delete
mphDeletionsList = new QList<QString>;
sphDeletionsList = new QList<QString>;
}
if (mphFound)
mphDeletionsList->append(hb->getIndex());
else
sphDeletionsList->append(hb->getIndex());
continue;
}
if (m_cancel)
break;
hl->headerTreeModel->setupTopLevelItem(hb);
++count;
if (count % 50 == 0)
{
QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
" of " + QString("%L1").arg(ng->getTotal()) +
" articles.";
emit progress(job->seq, count, labelText);
QCoreApplication::processEvents();
}
}
if (m_cancel)
break;
}
if ((t_ret = dbcp->close(dbcp)) != 0)
{
ng->getDb()->err(ret, "DBcursor->close");
if (ret == 0)
ret = t_ret;
}
emit loadReady(job->seq);
QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
" of " + QString("%L1").arg(ng->getTotal()) +
" articles.";
emit progress(job->seq, count, labelText);
QCoreApplication::processEvents();
if (!m_cancel)
{
quint64 totalArticles = ng->getTotal();
if (totalArticles != count || ng->unread() > count)
{
qint64 difference = count - totalArticles; // may be negative
quint64 unread = (quint64)qMax((qint64)(ng->unread() + difference), (qint64)0);
if (unread > count)
unread = count;
ng->setTotal(count);
ng->setUnread(unread);
emit updateArticleCounts(ng);
}
}
qDebug() << "Loading finished. Seconds: " << start.secsTo(QTime::currentTime());
qDebug() << "Ignored " << numIgnored << " articles";
qDebug() << "Loaded " << count << " articles";
if (mphDeletionsList) // need bulk deletion
emit addBulkJob(JobList::BulkDelete, ng, hl, mphDeletionsList, sphDeletionsList);
if (m_cancel)
{
emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkLoad, JobList::Finished_Ok, job->seq);
return true;
}
static void bulk_get_hash (u_int32_t offset, u_int32_t count, bdb_drv_t *pdrv) {
int ret;
size_t retklen, retdlen;
void *retkey, *retdata;
void* p;
DB* pdb;
DBC* pdbc = NULL;
DBT key, data;
db_recno_t curr = 1;
db_recno_t limit = 1;
db_recno_t actual_count = 0;
ErlDrvTermData *spec;
u_int32_t spec_items, idx;
curr = offset;
limit = offset + count;
pdb = pdrv->pcfg->pdb;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
data.data = pdrv->pcfg->buffer;
data.ulen = pdrv->pcfg->bulk_get_buffer_size_bytes;
data.flags = DB_DBT_USERMEM;
if ((ret = pdb->cursor(pdb, NULL, &pdbc, 0)) != 0) {
return_error_tuple(pdrv, db_strerror(ret));
return;
}
//Fast forward to the correct index
idx = 1;
while ((ret = pdbc->c_get(pdbc, &key, &data, DB_NEXT)) == 0) {
idx++;
if (idx >= offset) { break; }
}
if ((ret != DB_NOTFOUND) && (ret != 0)) {
return_error_tuple(pdrv, db_strerror(ret));
} else if (ret == DB_NOTFOUND) {
return_ok_empty_list(pdrv);
} else {
if ((ret = pdbc->c_get(pdbc, &key, &data, DB_MULTIPLE_KEY | DB_CURRENT)) != 0) {
if (ret == DB_NOTFOUND) {
return_ok_empty_list(pdrv);
} else {
return_error_tuple(pdrv, db_strerror(ret));
}
} else {
//First count the number of recs...
for (DB_MULTIPLE_INIT(p, &data); curr < limit; curr++) {
DB_MULTIPLE_KEY_NEXT(p, &data, retkey, retklen, retdata, retdlen);
if (p == NULL) {
break;
} else {
actual_count++;
}
}
spec_items = (8 * actual_count) + 7;
spec = malloc(sizeof(ErlDrvTermData) * spec_items);
if (spec == NULL) {
return_error_tuple(pdrv, "Could not allocate memory for operation!");
} else {
spec[0] = ERL_DRV_ATOM;
spec[1] = driver_mk_atom("ok");
//Now pipe the data...
p = NULL;
curr = 0;
for (DB_MULTIPLE_INIT(p, &data); curr < actual_count; curr++) {
DB_MULTIPLE_KEY_NEXT(p, &data, retkey, retklen, retdata, retdlen);
if (p == NULL) {
break;
} else {
idx = 2 + (curr * 8);
spec[idx + 0] = ERL_DRV_STRING;
spec[idx + 1] = (ErlDrvTermData)retkey;
spec[idx + 2] = retklen;
spec[idx + 3] = ERL_DRV_STRING;
spec[idx + 4] = (ErlDrvTermData)retdata;
spec[idx + 5] = retdlen;
spec[idx + 6] = ERL_DRV_TUPLE;
spec[idx + 7] = 2;
}
}
spec[spec_items - 5] = ERL_DRV_NIL;
spec[spec_items - 4] = ERL_DRV_LIST;
spec[spec_items - 3] = actual_count + 1;
spec[spec_items - 2] = ERL_DRV_TUPLE;
spec[spec_items - 1] = 2;
#if ((ERL_DRV_EXTENDED_MAJOR_VERSION == 1) || ((ERL_DRV_EXTENDED_MAJOR_VERSION == 2) && (ERL_DRV_EXTENDED_MINOR_VERSION == 0)))
driver_output_term(pdrv->port, spec, spec_items);
#else
ErlDrvTermData mkport = driver_mk_port(pdrv->port);
erl_drv_output_term(mkport, spec, spec_items);
#endif
free(spec);
}
}
}
pdbc->c_close(pdbc);
return;
}
int puavo_conf_db_get_all(struct puavo_conf *const conf,
struct puavo_conf_list *const list,
struct puavo_conf_err *const errp)
{
DBC *db_cursor = NULL;
DBT db_null;
DBT db_batch;
size_t length = 0;
char **keys = NULL;
char **values = NULL;
int ret = -1;
int db_error;
memset(&db_null, 0, sizeof(DBT));
memset(&db_batch, 0, sizeof(DBT));
db_batch.flags = DB_DBT_USERMEM;
db_batch.ulen = PUAVO_CONF_DEFAULT_DB_BATCH_SIZE;
db_batch.data = malloc(db_batch.ulen);
if (!db_batch.data) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS, 0,
"Failed to get all parameters");
goto out;
}
db_error = conf->db->cursor(conf->db, NULL, &db_cursor, 0);
if (db_error) {
db_cursor = NULL;
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
goto out;
}
/* Iterate key/value pairs in batches until all are found. */
while (1) {
void *batch_iterator;
/* Get the next batch of key-value pairs. */
db_error = db_cursor->get(db_cursor, &db_null, &db_batch,
DB_MULTIPLE_KEY | DB_NEXT);
switch (db_error) {
case 0:
break;
case DB_NOTFOUND:
ret = 0;
goto out;
default:
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
goto out;
}
/* Iterate the batch. */
DB_MULTIPLE_INIT(batch_iterator, &db_batch);
while (1) {
char *key;
char *val;
size_t key_size;
size_t val_size;
char **new_keys;
char **new_values;
DB_MULTIPLE_KEY_NEXT(batch_iterator, &db_batch,
key, key_size, val, val_size);
if (!batch_iterator)
break; /* The batch is empty. */
new_keys = realloc(keys,
sizeof(char *) * (length + 1));
if (!new_keys) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS,
0, "Failed to get all "
"parameters");
goto out;
}
keys = new_keys;
new_values = realloc(values,
sizeof(char *) * (length + 1));
if (!new_values) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS,
0, "Failed to get all "
"parameters");
goto out;
}
values = new_values;
++length;
keys[length - 1] = strndup(key, key_size);
values[length - 1] = strndup(val, val_size);
}
}
ret = 0;
out:
if (db_cursor) {
db_error = db_cursor->close(db_cursor);
/* Obey exit-on-first-error policy: Do not shadow any
* existing error, record close error only if we are
* cleaning up without any earlier errors. */
if (!ret && db_error) {
ret = -1;
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
}
}
if (ret) {
size_t i;
for (i = 0; i < length; ++i) {
free(keys[i]);
free(values[i]);
}
free(keys);
free(values);
} else {
list->keys = keys;
list->values = values;
list->length = length;
}
free(db_batch.data);
return ret;
}
