int db_cursor_get_multiple_key(DBC *cursor,
unsigned char *keybuf, u_int32_t keybuf_size,
u_int32_t keybuf_length,
unsigned char *buffer, u_int32_t buffer_size,
u_int32_t buffer_length,
u_int32_t flags, u_int32_t *ret_key_size,
u_int32_t *result_size,
void **pointer, DBT **data) {
DBT DBTKey, DBTValue;
int ret;
memset(&DBTKey, 0, sizeof(DBT));
memset(&DBTValue, 0, sizeof(DBT));
DBTKey.data = keybuf;
DBTKey.size = keybuf_size;
DBTKey.ulen = keybuf_length;
DBTKey.flags |= DB_DBT_USERMEM;
DBTValue.data = buffer;
DBTValue.size = buffer_size;
DBTValue.ulen = buffer_length;
DBTValue.flags |= DB_DBT_USERMEM;
flags |= DB_MULTIPLE_KEY;
ret = cursor->c_get(cursor, &DBTKey, &DBTValue, flags);
*ret_key_size = DBTKey.size;
*result_size = DBTValue.size;
if ((DBTKey.size <= DBTKey.ulen) && (DBTValue.size <= DBTValue.ulen)) {
**data = DBTValue;
DB_MULTIPLE_INIT(*pointer, *data);
}
return ret;
}
void kmapdset_iterator_impl_base::bulk_load(DBT* tk1)
{
NARK_RT_assert(0 == m_ret, std::logic_error);
load_key1(tk1->data, tk1->size);
clear_vec();
int ret;
do {
void *bptr, *data;
DB_MULTIPLE_INIT(bptr, &m_bulk);
assert(NULL != bptr);
for (;;)
{
size_t size = 0;
DB_MULTIPLE_NEXT(bptr, &m_bulk, data, size);
if (nark_likely(NULL != bptr))
this->push_back(data, size);
else
break;
}
ret = m_curp->get(m_curp, tk1, &m_bulk, DB_MULTIPLE|DB_NEXT_DUP);
} while (0 == ret);
}
int
bdb_idl_fetch_key(
BackendDB *be,
DB *db,
DB_TXN *txn,
DBT *key,
ID *ids,
DBC **saved_cursor,
int get_flag )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
int rc;
DBT data, key2, *kptr;
DBC *cursor;
ID *i;
void *ptr;
size_t len;
int rc2;
int flags = bdb->bi_db_opflags | DB_MULTIPLE;
int opflag;
/* If using BerkeleyDB 4.0, the buf must be large enough to
* grab the entire IDL in one get(), otherwise BDB will leak
* resources on subsequent get's. We can safely call get()
* twice - once for the data, and once to get the DB_NOTFOUND
* result meaning there's no more data. See ITS#2040 for details.
* This bug is fixed in BDB 4.1 so a smaller buffer will work if
* stack space is too limited.
*
* configure now requires Berkeley DB 4.1.
*/
#if DB_VERSION_FULL < 0x04010000
# define BDB_ENOUGH 5
#else
/* We sometimes test with tiny IDLs, and BDB always wants buffers
* that are at least one page in size.
*/
# if BDB_IDL_DB_SIZE < 4096
# define BDB_ENOUGH 2048
# else
# define BDB_ENOUGH 1
# endif
#endif
ID buf[BDB_IDL_DB_SIZE*BDB_ENOUGH];
char keybuf[16];
Debug( LDAP_DEBUG_ARGS,
"bdb_idl_fetch_key: %s\n",
bdb_show_key( key, keybuf ), 0, 0 );
assert( ids != NULL );
if ( saved_cursor && *saved_cursor ) {
opflag = DB_NEXT;
} else if ( get_flag == LDAP_FILTER_GE ) {
opflag = DB_SET_RANGE;
} else if ( get_flag == LDAP_FILTER_LE ) {
opflag = DB_FIRST;
} else {
opflag = DB_SET;
}
/* only non-range lookups can use the IDL cache */
if ( bdb->bi_idl_cache_size && opflag == DB_SET ) {
rc = bdb_idl_cache_get( bdb, db, key, ids );
if ( rc != LDAP_NO_SUCH_OBJECT ) return rc;
}
DBTzero( &data );
data.data = buf;
data.ulen = sizeof(buf);
data.flags = DB_DBT_USERMEM;
/* If we're not reusing an existing cursor, get a new one */
if( opflag != DB_NEXT ) {
rc = db->cursor( db, txn, &cursor, bdb->bi_db_opflags );
if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"cursor failed: %s (%d)\n", db_strerror(rc), rc, 0 );
return rc;
}
} else {
cursor = *saved_cursor;
}
/* If this is a LE lookup, save original key so we can determine
* when to stop. If this is a GE lookup, save the key since it
* will be overwritten.
*/
if ( get_flag == LDAP_FILTER_LE || get_flag == LDAP_FILTER_GE ) {
DBTzero( &key2 );
key2.flags = DB_DBT_USERMEM;
key2.ulen = sizeof(keybuf);
key2.data = keybuf;
key2.size = key->size;
AC_MEMCPY( keybuf, key->data, key->size );
kptr = &key2;
} else {
kptr = key;
}
len = key->size;
rc = cursor->c_get( cursor, kptr, &data, flags | opflag );
/* skip presence key on range inequality lookups */
while (rc == 0 && kptr->size != len) {
rc = cursor->c_get( cursor, kptr, &data, flags | DB_NEXT_NODUP );
}
/* If we're doing a LE compare and the new key is greater than
* our search key, we're done
*/
if (rc == 0 && get_flag == LDAP_FILTER_LE && memcmp( kptr->data,
key->data, key->size ) > 0 ) {
rc = DB_NOTFOUND;
}
if (rc == 0) {
i = ids;
while (rc == 0) {
u_int8_t *j;
DB_MULTIPLE_INIT( ptr, &data );
while (ptr) {
DB_MULTIPLE_NEXT(ptr, &data, j, len);
if (j) {
++i;
BDB_DISK2ID( j, i );
}
}
rc = cursor->c_get( cursor, key, &data, flags | DB_NEXT_DUP );
}
if ( rc == DB_NOTFOUND ) rc = 0;
ids[0] = i - ids;
/* On disk, a range is denoted by 0 in the first element */
if (ids[1] == 0) {
if (ids[0] != BDB_IDL_RANGE_SIZE) {
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"range size mismatch: expected %d, got %ld\n",
BDB_IDL_RANGE_SIZE, ids[0], 0 );
cursor->c_close( cursor );
return -1;
}
BDB_IDL_RANGE( ids, ids[2], ids[3] );
}
data.size = BDB_IDL_SIZEOF(ids);
}
if ( saved_cursor && rc == 0 ) {
if ( !*saved_cursor )
*saved_cursor = cursor;
rc2 = 0;
}
else
rc2 = cursor->c_close( cursor );
if (rc2) {
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"close failed: %s (%d)\n", db_strerror(rc2), rc2, 0 );
return rc2;
}
if( rc == DB_NOTFOUND ) {
return rc;
} else if( rc != 0 ) {
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"get failed: %s (%d)\n",
db_strerror(rc), rc, 0 );
return rc;
} else if ( data.size == 0 || data.size % sizeof( ID ) ) {
/* size not multiple of ID size */
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"odd size: expected %ld multiple, got %ld\n",
(long) sizeof( ID ), (long) data.size, 0 );
return -1;
} else if ( data.size != BDB_IDL_SIZEOF(ids) ) {
/* size mismatch */
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"get size mismatch: expected %ld, got %ld\n",
(long) ((1 + ids[0]) * sizeof( ID )), (long) data.size, 0 );
return -1;
}
if ( bdb->bi_idl_cache_max_size ) {
bdb_idl_cache_put( bdb, db, key, ids, rc );
}
return rc;
}
static int
hdb_dn2idl_internal(
struct dn2id_cookie *cx
)
{
BDB_IDL_ZERO( cx->tmp );
if ( cx->bdb->bi_idl_cache_size ) {
char *ptr = ((char *)&cx->id)-1;
cx->key.data = ptr;
cx->key.size = sizeof(ID)+1;
if ( cx->prefix == DN_SUBTREE_PREFIX ) {
ID *ids = cx->depth ? cx->tmp : cx->ids;
*ptr = cx->prefix;
cx->rc = bdb_idl_cache_get(cx->bdb, cx->db, &cx->key, ids);
if ( cx->rc == LDAP_SUCCESS ) {
if ( cx->depth ) {
bdb_idl_delete( cx->tmp, cx->id ); /* ITS#6983, drop our own ID */
bdb_idl_append( cx->ids, cx->tmp );
cx->need_sort = 1;
}
return cx->rc;
}
}
*ptr = DN_ONE_PREFIX;
cx->rc = bdb_idl_cache_get(cx->bdb, cx->db, &cx->key, cx->tmp);
if ( cx->rc == LDAP_SUCCESS ) {
goto gotit;
}
if ( cx->rc == DB_NOTFOUND ) {
return cx->rc;
}
}
bdb_cache_entryinfo_lock( cx->ei );
/* If number of kids in the cache differs from on-disk, load
* up all the kids from the database
*/
if ( cx->ei->bei_ckids+1 != cx->ei->bei_dkids ) {
EntryInfo ei;
db_recno_t dkids = cx->ei->bei_dkids;
ei.bei_parent = cx->ei;
/* Only one thread should load the cache */
while ( cx->ei->bei_state & CACHE_ENTRY_ONELEVEL ) {
bdb_cache_entryinfo_unlock( cx->ei );
ldap_pvt_thread_yield();
bdb_cache_entryinfo_lock( cx->ei );
if ( cx->ei->bei_ckids+1 == cx->ei->bei_dkids ) {
goto synced;
}
}
cx->ei->bei_state |= CACHE_ENTRY_ONELEVEL;
bdb_cache_entryinfo_unlock( cx->ei );
cx->rc = cx->db->cursor( cx->db, NULL, &cx->dbc,
cx->bdb->bi_db_opflags );
if ( cx->rc )
goto done_one;
cx->data.data = &cx->dbuf;
cx->data.ulen = sizeof(ID);
cx->data.dlen = sizeof(ID);
cx->data.flags = DB_DBT_USERMEM | DB_DBT_PARTIAL;
/* The first item holds the parent ID. Ignore it. */
cx->key.data = &cx->nid;
cx->key.size = sizeof(ID);
cx->rc = cx->dbc->c_get( cx->dbc, &cx->key, &cx->data, DB_SET );
if ( cx->rc ) {
cx->dbc->c_close( cx->dbc );
goto done_one;
}
/* If the on-disk count is zero we've never checked it.
* Count it now.
*/
if ( !dkids ) {
cx->dbc->c_count( cx->dbc, &dkids, 0 );
cx->ei->bei_dkids = dkids;
}
cx->data.data = cx->buf;
cx->data.ulen = BDB_IDL_UM_SIZE * sizeof(ID);
cx->data.flags = DB_DBT_USERMEM;
if ( dkids > 1 ) {
/* Fetch the rest of the IDs in a loop... */
while ( (cx->rc = cx->dbc->c_get( cx->dbc, &cx->key, &cx->data,
DB_MULTIPLE | DB_NEXT_DUP )) == 0 ) {
u_int8_t *j;
size_t len;
void *ptr;
DB_MULTIPLE_INIT( ptr, &cx->data );
while (ptr) {
DB_MULTIPLE_NEXT( ptr, &cx->data, j, len );
if (j) {
EntryInfo *ei2;
diskNode *d = (diskNode *)j;
short nrlen;
BDB_DISK2ID( j + len - sizeof(ID), &ei.bei_id );
nrlen = ((d->nrdnlen[0] ^ 0x80) << 8) | d->nrdnlen[1];
ei.bei_nrdn.bv_len = nrlen;
/* nrdn/rdn are set in-place.
* hdb_cache_load will copy them as needed
*/
ei.bei_nrdn.bv_val = d->nrdn;
ei.bei_rdn.bv_len = len - sizeof(diskNode)
- ei.bei_nrdn.bv_len;
ei.bei_rdn.bv_val = d->nrdn + ei.bei_nrdn.bv_len + 1;
bdb_idl_append_one( cx->tmp, ei.bei_id );
hdb_cache_load( cx->bdb, &ei, &ei2 );
}
}
}
}
cx->rc = cx->dbc->c_close( cx->dbc );
done_one:
bdb_cache_entryinfo_lock( cx->ei );
cx->ei->bei_state &= ~CACHE_ENTRY_ONELEVEL;
bdb_cache_entryinfo_unlock( cx->ei );
if ( cx->rc )
return cx->rc;
} else {
/* The in-memory cache is in sync with the on-disk data.
* do we have any kids?
*/
synced:
cx->rc = 0;
if ( cx->ei->bei_ckids > 0 ) {
/* Walk the kids tree; order is irrelevant since bdb_idl_sort
* will sort it later.
*/
avl_apply( cx->ei->bei_kids, apply_func,
cx->tmp, -1, AVL_POSTORDER );
}
bdb_cache_entryinfo_unlock( cx->ei );
}
if ( !BDB_IDL_IS_RANGE( cx->tmp ) && cx->tmp[0] > 3 )
bdb_idl_sort( cx->tmp, cx->buf );
if ( cx->bdb->bi_idl_cache_max_size && !BDB_IDL_IS_ZERO( cx->tmp )) {
char *ptr = ((char *)&cx->id)-1;
cx->key.data = ptr;
cx->key.size = sizeof(ID)+1;
*ptr = DN_ONE_PREFIX;
bdb_idl_cache_put( cx->bdb, cx->db, &cx->key, cx->tmp, cx->rc );
}
gotit:
if ( !BDB_IDL_IS_ZERO( cx->tmp )) {
if ( cx->prefix == DN_SUBTREE_PREFIX ) {
bdb_idl_append( cx->ids, cx->tmp );
cx->need_sort = 1;
if ( !(cx->ei->bei_state & CACHE_ENTRY_NO_GRANDKIDS)) {
ID *save, idcurs;
EntryInfo *ei = cx->ei;
int nokids = 1;
save = cx->op->o_tmpalloc( BDB_IDL_SIZEOF( cx->tmp ),
cx->op->o_tmpmemctx );
BDB_IDL_CPY( save, cx->tmp );
idcurs = 0;
cx->depth++;
for ( cx->id = bdb_idl_first( save, &idcurs );
cx->id != NOID;
cx->id = bdb_idl_next( save, &idcurs )) {
EntryInfo *ei2;
cx->ei = NULL;
if ( bdb_cache_find_id( cx->op, cx->txn, cx->id, &cx->ei,
ID_NOENTRY, NULL ))
continue;
if ( cx->ei ) {
ei2 = cx->ei;
if ( !( ei2->bei_state & CACHE_ENTRY_NO_KIDS )) {
BDB_ID2DISK( cx->id, &cx->nid );
hdb_dn2idl_internal( cx );
if ( !BDB_IDL_IS_ZERO( cx->tmp ))
nokids = 0;
}
bdb_cache_entryinfo_lock( ei2 );
ei2->bei_finders--;
bdb_cache_entryinfo_unlock( ei2 );
}
}
cx->depth--;
cx->op->o_tmpfree( save, cx->op->o_tmpmemctx );
if ( nokids ) {
bdb_cache_entryinfo_lock( ei );
ei->bei_state |= CACHE_ENTRY_NO_GRANDKIDS;
bdb_cache_entryinfo_unlock( ei );
}
}
/* Make sure caller knows it had kids! */
cx->tmp[0]=1;
cx->rc = 0;
} else {
BDB_IDL_CPY( cx->ids, cx->tmp );
}
}
return cx->rc;
}
static void
b_inmem_op_ds_bulk(u_int ops, u_int *totalp)
{
DB_ENV *dbenv;
DB *dbp;
DBC *dbc;
DBT key, data;
u_int32_t len, klen;
u_int i, total;
char *keybuf, *databuf;
void *pointer, *dp, *kp;
DB_MPOOL_STAT *gsp;
DB_BENCH_ASSERT((keybuf = malloc(keysize)) != NULL);
DB_BENCH_ASSERT((databuf = malloc(bulkbufsize)) != NULL);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
key.data = keybuf;
key.size = keysize;
data.data = databuf;
data.size = datasize;
memset(databuf, 'b', datasize);
DB_BENCH_ASSERT(db_create(&dbp, NULL, 0) == 0);
dbenv = dbp->dbenv;
dbp->set_errfile(dbp, stderr);
DB_BENCH_ASSERT(dbp->set_pagesize(dbp, pagesize) == 0);
DB_BENCH_ASSERT(dbp->set_cachesize(dbp, 0, cachesize, 1) == 0);
DB_BENCH_ASSERT(
dbp->open(dbp, NULL, NULL, NULL, DB_BTREE, DB_CREATE, 0666) == 0);
for (i = 1; i <= numitems; ++i) {
(void)snprintf(keybuf, keysize, "%7d", i);
DB_BENCH_ASSERT(dbp->put(dbp, NULL, &key, &data, 0) == 0);
}
#if 0
fp = fopen("before", "w");
dbp->set_msgfile(dbp, fp);
DB_BENCH_ASSERT (dbp->stat_print(dbp, DB_STAT_ALL) == 0);
#endif
DB_BENCH_ASSERT(dbp->cursor(dbp, NULL, &dbc, 0) == 0);
data.ulen = bulkbufsize;
data.flags = DB_DBT_USERMEM;
(void)dbenv->memp_stat(dbenv, &gsp, NULL, DB_STAT_CLEAR);
TIMER_START;
for (total = 0; ops > 0; --ops) {
DB_BENCH_ASSERT(dbc->c_get(
dbc, &key, &data, DB_FIRST | DB_MULTIPLE_KEY) == 0);
DB_MULTIPLE_INIT(pointer, &data);
while (pointer != NULL) {
DB_MULTIPLE_KEY_NEXT(pointer, &data, kp, klen, dp, len);
if (kp != NULL)
++total;
}
}
TIMER_STOP;
*totalp = total;
if (dbenv->memp_stat(dbenv, &gsp, NULL, 0) == 0)
DB_BENCH_ASSERT(gsp->st_cache_miss == 0);
#if 0
fp = fopen("before", "w");
dbp->set_msgfile(dbp, fp);
DB_BENCH_ASSERT (dbp->stat_print(dbp, DB_STAT_ALL) == 0);
#endif
DB_BENCH_ASSERT(dbp->close(dbp, 0) == 0);
COMPQUIET(dp, NULL);
COMPQUIET(klen, 0);
COMPQUIET(len, 0);
}
bool BulkHeaderGroup::BulkHeaderGroupBody()
{
// belt and braces
key.set_data(keymem);
data.set_data(datamem);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkHeaderGroup, JobList::Running, job->seq);
NewsGroup* ng = job->ng;
Db* db = ng->getDb();
MultiPartHeader mph;
SinglePartHeader sph;
HeaderBase* hb = 0;
HeaderGroup* headerGroup = 0;
HeaderGroup* advancedHeaderGroup = 0;
// typedef QMap<QString, QString> HeaderGroupIndexes; // subj, headerGroup index
// typedef QMap<QString, HeaderGroup*> HeaderGroups; // headerGroup index, headerGroup *
HeaderGroupIndexes headerGroupIndexes;
HeaderGroups headerGroups;
DBC *dbcp = 0;
DBT ckey, cdata;
memset(&ckey, 0, sizeof(ckey));
memset(&cdata, 0, sizeof(cdata));
size_t retklen, retdlen;
void *retkey = 0, *retdata = 0;
int ret, t_ret;
void *p = 0;
quint64 count=0;
cdata.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
cdata.flags = DB_DBT_USERMEM;
ckey.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
ckey.flags = DB_DBT_USERMEM;
/* Acquire a cursor for the database. */
if ((ret = db->get_DB()->cursor(db->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
{
db->err(ret, "DB->cursor");
char* ptr = 0;
ptr = (char*)(ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = (char*)(cdata.data);
Q_DELETE_ARRAY(ptr);
return false;
}
// To save the group records
ng->articlesNeedDeleting(false);
// Store the data in the database - flush first ...
u_int32_t delCount;
uchar keymem[KEYMEM_SIZE];
uchar datamem[DATAMEM_SIZE];
Dbt key, data;
char* p2 = 0;
QByteArray ba;
const char *k = 0;
key.set_flags(DB_DBT_USERMEM);
key.set_data(&keymem);
key.set_ulen(KEYMEM_SIZE);
data.set_flags(DB_DBT_USERMEM);
data.set_ulen(DATAMEM_SIZE);
data.set_data(&datamem);
QString subj = "MDQuban", from = "MDQuban";
//QString rs1 = "^(.*)(\".*\")";
//QString rs2 = "^(.*)\\s-\\s(.*)$";
//QString rs3 = "^(\\S+.*)\\[.*\\].*(\".*\")";
//QString rs3 = "^(.*)\\s-\\s.*\\s-\\s(.*)$";
QRegExp rx[3];
bool rxPosBack[3];
bool noRegexpGrouping;
QString recKey, storeIndex;
QString prevSubj = "MDQuban", prevFrom = "MDQuban";
int pos;
bool newGroup = false;
bool mphFound = false;
quint32 grouped = 0,
single = 0,
numGroups = 0;
qint16 stringDiff = -1;
bool prevGroup = false;
bool advancedPlacement = false;
bool skipAdvanced = false;
noRegexpGrouping = ng->isThereNoRegexOnGrouping();
if (noRegexpGrouping == false) // need regex for grouping
{
rx[0].setPattern(ng->getGroupRE1());
rx[1].setPattern(ng->getGroupRE2());
rx[2].setPattern(ng->getGroupRE3());
rxPosBack[0] = ng->getGroupRE1Back();
rxPosBack[1] = ng->getGroupRE2Back();
rxPosBack[2] = ng->getGroupRE3Back();
}
ng->getGroupingDb()->truncate(0, &delCount, 0);
qDebug() << "Deleted " << delCount << " records from group db";
QMapIterator<QString, QString> it(headerGroupIndexes);
QString advancedIndex;
for (;;)
{
/*
* Acquire the next set of key/data pairs. This code
* does not handle single key/data pairs that won't fit
* in a BUFFER_LENGTH size buffer, instead returning
* DB_BUFFER_SMALL to our caller.
*/
if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
{
if (ret != DB_NOTFOUND)
db->err(ret, "DBcursor->get");
break;
}
for (DB_MULTIPLE_INIT(p, &cdata);;)
{
DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
if (retdlen){;} // MD TODO compiler .... unused variable
recKey = QString::fromLocal8Bit((char*)retkey, retklen);
if (*((char *)retdata) == 'm')
{
MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
hb = (HeaderBase*)&mph;
mphFound = true;
}
else if (*((char *)retdata) == 's')
{
SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
hb = (HeaderBase*)&sph;
mphFound = false;
}
else
{
// What have we found ?????
qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
continue;
}
++count;
prevSubj = subj;
prevFrom = from;
subj = hb->getSubj();
from = hb->getFrom();
if (noRegexpGrouping == false) // need regex for grouping
{
for (int i=0; i<3; ++i)
{
if (rx[i].isEmpty() == false)
{
if (rxPosBack[i] == true) // from the back
{
pos = subj.lastIndexOf(rx[i]);
if (pos != -1)
subj.truncate(pos);
}
else // from the front
{
pos = rx[i].indexIn(subj);
if (pos > -1)
subj = rx[i].cap(0);
}
}
}
}
//qDebug() << "Stripped down to: " << subj;
stringDiff = -1;
if (prevFrom != from) // change of contributor
{
newGroup = true;
}
else // same contributor
{
if ((stringDiff = levenshteinDistance(prevSubj, subj)) > ng->getMatchDistance()) // no match ...
newGroup = true;
else
newGroup = false;
//qDebug() << "Diff between " << prevSubj << " and " << subj << " is " << stringDiff;
}
if (newGroup)
{
if (ng->isThereAdvancedGrouping())
{
it.toFront();
// decide if we can match to a previous group
while (it.hasNext())
{
it.next();
if ((stringDiff = levenshteinDistance(it.key(), subj)) <= ng->getMatchDistance()) // match ...
{
// The index for this group is in it.value()
// See if we have the HeaderGroup in our cache headerGroups)
if (headerGroups.contains(it.value()))
{
advancedHeaderGroup = headerGroups.value(it.value());
}
else // not in cache
{
advancedIndex = it.value();
advancedHeaderGroup = getGroup(ng, advancedIndex);
if (advancedHeaderGroup)
{
headerGroups.insert(advancedIndex, advancedHeaderGroup);
}
else // db read failed ..
{
skipAdvanced = true;
}
}
if (skipAdvanced == false)
{
if (mphFound)
advancedHeaderGroup->addMphKey(recKey);
else
advancedHeaderGroup->addSphKey(recKey);
advancedPlacement = true;
subj = prevSubj; // ignore this header as it's been placed out of sequence
from = prevFrom;
newGroup = false; // as we managed to relocate to an existing group
break; // stop looking at previous groups
}
else
skipAdvanced = false;
}
}
}
}
if (newGroup)
{
if (prevGroup) // save before moving on
{
ba = storeIndex.toLocal8Bit();
k= ba.constData();
memcpy(keymem, k, storeIndex.length());
key.set_size(storeIndex.length());
p2=headerGroup->data();
data.set_data(p2);
data.set_size(headerGroup->getRecordSize());
ret=ng->getGroupingDb()->put(NULL, &key, &data, 0);
if (ret!=0)
qDebug("Error updating record: %d", ret);
if (ng->isThereAdvancedGrouping())
headerGroupIndexes.insert(storeIndex.section('\n', 0, 0), storeIndex);
Q_DELETE_ARRAY(p2);
Q_DELETE(headerGroup);
numGroups++;
}
prevGroup = true;
storeIndex = subj % "\n" % from;
headerGroup = new HeaderGroup();
headerGroup->setDisplayName(subj);
headerGroup->setPostingDate(hb->getPostingDate());
headerGroup->setDownloadDate(hb->getDownloadDate());
headerGroup->setStatus(hb->getStatus());
headerGroup->setNextDistance(stringDiff);
}
// if we've found somewhere else to place this header then don't add again
if (!advancedPlacement)
{
if (mphFound)
headerGroup->addMphKey(recKey);
else
headerGroup->addSphKey(recKey);
}
else
advancedPlacement = false;
if (count % 250 == 0)
{
QCoreApplication::processEvents();
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if ((t_ret = dbcp->close(dbcp)) != 0)
{
db->err(ret, "DBcursor->close");
if (ret == 0)
ret = t_ret;
}
char* ptr = ((char*)ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = ((char*)cdata.data);
Q_DELETE_ARRAY(ptr);
if (headerGroups.count())
{
qDeleteAll(headerGroups);
headerGroups.clear();
}
qDebug() << "Multi = " << grouped << ", single = " << single;
ng->setHeadersNeedGrouping(false);
// Finally update the newsgroup
emit saveGroup(ng);
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit logEvent(tr("Bulk grouping of ") + ng->getTotal() + tr(" articles completed successfully."));
emit updateJob(JobList::BulkHeaderGroup, JobList::Finished_Ok, job->seq);
ng->setTotalGroups(numGroups);
Q_DELETE(headerGroup);
return true;
}
int
bdb_idl_fetch_key(
BackendDB *be,
DB *db,
DB_TXN *tid,
DBT *key,
ID *ids )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
int rc;
DBT data;
DBC *cursor;
ID *i;
void *ptr;
size_t len;
int rc2;
int flags = bdb->bi_db_opflags | DB_MULTIPLE;
#ifdef SLAP_IDL_CACHE
bdb_idl_cache_entry_t idl_tmp;
#endif
/* If using BerkeleyDB 4.0, the buf must be large enough to
* grab the entire IDL in one get(), otherwise BDB will leak
* resources on subsequent get's. We can safely call get()
* twice - once for the data, and once to get the DB_NOTFOUND
* result meaning there's no more data. See ITS#2040 for details.
* This bug is fixed in BDB 4.1 so a smaller buffer will work if
* stack space is too limited.
*
* configure now requires Berkeley DB 4.1.
*/
#if (DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR == 0)
# define BDB_ENOUGH 5
#else
# define BDB_ENOUGH 1
#endif
ID buf[BDB_IDL_DB_SIZE*BDB_ENOUGH];
char keybuf[16];
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ARGS,
"bdb_idl_fetch_key: %s\n",
bdb_show_key( key, keybuf ), 0, 0 );
#else
Debug( LDAP_DEBUG_ARGS,
"bdb_idl_fetch_key: %s\n",
bdb_show_key( key, keybuf ), 0, 0 );
#endif
assert( ids != NULL );
#ifdef SLAP_IDL_CACHE
if ( bdb->bi_idl_cache_max_size ) {
bdb_idl_cache_entry_t *matched_idl_entry;
DBT2bv( key, &idl_tmp.kstr );
idl_tmp.db = db;
ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_mutex );
matched_idl_entry = avl_find( bdb->bi_idl_tree, &idl_tmp,
bdb_idl_entry_cmp );
if ( matched_idl_entry != NULL ) {
BDB_IDL_CPY( ids, matched_idl_entry->idl );
IDL_LRU_DELETE( bdb, matched_idl_entry );
IDL_LRU_ADD( bdb, matched_idl_entry );
ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_mutex );
return LDAP_SUCCESS;
}
ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_mutex );
}
#endif
DBTzero( &data );
data.data = buf;
data.ulen = sizeof(buf);
data.flags = DB_DBT_USERMEM;
if ( tid ) flags |= DB_RMW;
rc = db->cursor( db, tid, &cursor, bdb->bi_db_opflags );
if( rc != 0 ) {
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: cursor failed: %s (%d)\n",
db_strerror(rc), rc, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"cursor failed: %s (%d)\n", db_strerror(rc), rc, 0 );
#endif
return rc;
}
rc = cursor->c_get( cursor, key, &data, flags | DB_SET );
if (rc == 0) {
i = ids;
while (rc == 0) {
u_int8_t *j;
DB_MULTIPLE_INIT( ptr, &data );
while (ptr) {
DB_MULTIPLE_NEXT(ptr, &data, j, len);
if (j) {
++i;
AC_MEMCPY( i, j, sizeof(ID) );
}
}
rc = cursor->c_get( cursor, key, &data, flags | DB_NEXT_DUP );
}
if ( rc == DB_NOTFOUND ) rc = 0;
ids[0] = i - ids;
/* On disk, a range is denoted by 0 in the first element */
if (ids[1] == 0) {
if (ids[0] != BDB_IDL_RANGE_SIZE) {
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"=> bdb_idl_fetch_key: range size mismatch: "
"expected %ld, got %ld\n",
BDB_IDL_RANGE_SIZE, ids[0], 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"range size mismatch: expected %d, got %ld\n",
BDB_IDL_RANGE_SIZE, ids[0], 0 );
#endif
cursor->c_close( cursor );
return -1;
}
BDB_IDL_RANGE( ids, ids[2], ids[3] );
}
data.size = BDB_IDL_SIZEOF(ids);
}
rc2 = cursor->c_close( cursor );
if (rc2) {
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: close failed: %s (%d)\n",
db_strerror(rc2), rc2, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"close failed: %s (%d)\n", db_strerror(rc2), rc2, 0 );
#endif
return rc2;
}
if( rc == DB_NOTFOUND ) {
return rc;
} else if( rc != 0 ) {
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: get failed: %s (%d)\n",
db_strerror(rc), rc, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"get failed: %s (%d)\n",
db_strerror(rc), rc, 0 );
#endif
return rc;
} else if ( data.size == 0 || data.size % sizeof( ID ) ) {
/* size not multiple of ID size */
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: odd size: expected %ld multiple, got %ld\n",
(long) sizeof( ID ), (long) data.size, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"odd size: expected %ld multiple, got %ld\n",
(long) sizeof( ID ), (long) data.size, 0 );
#endif
return -1;
} else if ( data.size != BDB_IDL_SIZEOF(ids) ) {
/* size mismatch */
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: get size mismatch: expected %ld, got %ld\n",
(long) ((1 + ids[0]) * sizeof( ID )), (long) data.size, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"get size mismatch: expected %ld, got %ld\n",
(long) ((1 + ids[0]) * sizeof( ID )), (long) data.size, 0 );
#endif
return -1;
}
#ifdef SLAP_IDL_CACHE
if ( bdb->bi_idl_cache_max_size ) {
bdb_idl_cache_entry_t *ee;
ee = (bdb_idl_cache_entry_t *) ch_malloc(
sizeof( bdb_idl_cache_entry_t ) );
ee->db = db;
ee->idl = (ID*) ch_malloc( BDB_IDL_SIZEOF ( ids ) );
ee->idl_lru_prev = NULL;
ee->idl_lru_next = NULL;
BDB_IDL_CPY( ee->idl, ids );
ber_dupbv( &ee->kstr, &idl_tmp.kstr );
ldap_pvt_thread_mutex_lock( &bdb->bi_idl_tree_mutex );
if ( avl_insert( &bdb->bi_idl_tree, (caddr_t) ee,
bdb_idl_entry_cmp, avl_dup_error ))
{
ch_free( ee->kstr.bv_val );
ch_free( ee->idl );
ch_free( ee );
} else {
IDL_LRU_ADD( bdb, ee );
if ( ++bdb->bi_idl_cache_size > bdb->bi_idl_cache_max_size ) {
int i = 0;
while ( bdb->bi_idl_lru_tail != NULL && i < 10 ) {
ee = bdb->bi_idl_lru_tail;
if ( avl_delete( &bdb->bi_idl_tree, (caddr_t) ee,
bdb_idl_entry_cmp ) == NULL ) {
#ifdef NEW_LOGGING
LDAP_LOG( INDEX, ERR,
"bdb_idl_fetch_key: AVL delete failed\n",
0, 0, 0 );
#else
Debug( LDAP_DEBUG_ANY, "=> bdb_idl_fetch_key: "
"AVL delete failed\n",
0, 0, 0 );
#endif
}
IDL_LRU_DELETE( bdb, ee );
i++;
--bdb->bi_idl_cache_size;
ch_free( ee->kstr.bv_val );
ch_free( ee->idl );
ch_free( ee );
}
}
}
ldap_pvt_thread_mutex_unlock( &bdb->bi_idl_tree_mutex );
}
#endif
return rc;
}
/*
* Use Bulk Get/Put to copy the given number of pages worth of
* records from the source database to the destination database,
* this function should be called until all tables are copied, at
* which point it will return SQLITE_DONE. Both Btrees need to
* have transactions before calling this function.
* p->pSrc - Source Btree
* p->tables - Contains a list of iTables to copy, gotten using
* btreeGetTables().
* p->currentTable - Index in tables of the current table being copied.
* p->srcCur - Cursor on the current source table being copied.
* p->pDest - Destiniation Btree.
* p->destCur - BtCursor on the destination table being copied into.
* pages - Number of pages worth of data to copy.
*/
static int btreeCopyPages(sqlite3_backup *p, int *pages)
{
DB *dbp;
DBT dataOut, dataIn;
char bufOut[MULTI_BUFSIZE], bufIn[MULTI_BUFSIZE];
int ret, rc, copied, srcIsDupIndex;
void *in, *out, *app;
ret = 0;
rc = SQLITE_OK;
dbp = NULL;
copied = 0;
memset(&dataOut, 0, sizeof(dataOut));
memset(&dataIn, 0, sizeof(dataIn));
dataOut.flags = DB_DBT_USERMEM;
dataIn.flags = DB_DBT_USERMEM;
dataOut.data = bufOut;
dataOut.ulen = sizeof(bufOut);
dataIn.data = bufIn;
dataIn.ulen = sizeof(bufIn);
while (*pages < 0 || *pages > copied) {
/* No tables left to copy */
if (p->tables[p->currentTable] == -1) {
u32 val;
/*
* Update the schema file format and largest rootpage
* in the meta data. Other meta data values should
* not be changed.
*/
sqlite3BtreeGetMeta(p->pSrc, 1, &val);
if (p->pSrc->db->errCode == SQLITE_BUSY) {
rc = SQLITE_BUSY;
goto err;
}
rc = sqlite3BtreeUpdateMeta(p->pDest, 1, val);
if (rc != SQLITE_OK)
goto err;
sqlite3BtreeGetMeta(p->pSrc, 3, &val);
if (p->pSrc->db->errCode == SQLITE_BUSY) {
rc = SQLITE_BUSY;
goto err;
}
rc = sqlite3BtreeUpdateMeta(p->pDest, 3, val);
if (rc != SQLITE_OK)
goto err;
ret = SQLITE_DONE;
goto err;
}
/* If not currently copying a table, get the next table. */
if (!p->srcCur) {
rc = btreeGetUserTable(p->pSrc, p->srcTxn, &dbp,
p->tables[p->currentTable]);
if (rc != SQLITE_OK)
goto err;
assert(dbp);
memset(&p->destCur, 0, sizeof(p->destCur));
/*
* Open a cursor on the destination table, this will
* create the table and allow the Btree to manage the
* DB object.
*/
sqlite3BtreeCursor(p->pDest, p->tables[p->currentTable],
1, dbp->app_private, &p->destCur);
if ((rc = p->destCur.error) != SQLITE_OK) {
app = dbp->app_private;
dbp->close(dbp, DB_NOSYNC);
if (app)
sqlite3DbFree(p->pSrcDb, app);
goto err;
}
/* Open a cursor on the source table. */
if ((ret = dbp->cursor(dbp,
p->srcTxn, &p->srcCur, 0)) != 0)
goto err;
dbp = 0;
}
srcIsDupIndex = isDupIndex((p->tables[p->currentTable] & 1) ?
BTREE_INTKEY : 0, p->pSrc->pBt->dbStorage,
p->srcCur->dbp->app_private, p->srcCur->dbp);
/*
* Copy the current table until the given number of
* pages is copied, or the entire table has been copied.
*/
while (*pages < 0 || *pages > copied) {
DBT key, data;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
/* Do a Bulk Get from the source table. */
ret = p->srcCur->get(p->srcCur, &key, &dataOut,
DB_NEXT | DB_MULTIPLE_KEY);
if (ret == DB_NOTFOUND)
break;
if (ret != 0)
goto err;
/* Copy the records into the Bulk buffer. */
DB_MULTIPLE_INIT(out, &dataOut);
DB_MULTIPLE_WRITE_INIT(in, &dataIn);
DB_MULTIPLE_KEY_NEXT(out, &dataOut, key.data,
key.size, data.data, data.size);
while (out) {
/*
* Have to translate the index formats if they
* are not the same.
*/
if (p->destCur.isDupIndex != srcIsDupIndex) {
if (srcIsDupIndex) {
p->destCur.key = key;
p->destCur.data = data;
if (!btreeCreateIndexKey(
&p->destCur)) {
rc = SQLITE_NOMEM;
goto err;
}
DB_MULTIPLE_KEY_WRITE_NEXT(in,
&dataIn,
p->destCur.index.data,
p->destCur.index.size,
p->destCur.data.data, 0);
} else {
/* Copy the key into the cursor
* index since spliting the key
* requires changing the
* internal memory.
*/
if (!allocateCursorIndex(
&p->destCur, key.size)) {
rc = SQLITE_NOMEM;
goto err;
}
memcpy(p->destCur.index.data,
key.data, key.size);
p->destCur.index.size =
key.size;
p->destCur.key.data =
p->destCur.index.data;
p->destCur.key.size =
p->destCur.index.size;
splitIndexKey(&p->destCur);
DB_MULTIPLE_KEY_WRITE_NEXT(
in, &dataIn,
p->destCur.key.data,
p->destCur.key.size,
p->destCur.data.data,
p->destCur.data.size);
}
} else
DB_MULTIPLE_KEY_WRITE_NEXT(in, &dataIn,
key.data, key.size,
data.data, data.size);
DB_MULTIPLE_KEY_NEXT(out, &dataOut,
key.data, key.size,
data.data, data.size);
}
/* Insert into the destination table. */
dbp = p->destCur.cached_db->dbp;
if ((ret = dbp->put(dbp, p->pDest->savepoint_txn,
&dataIn, 0, DB_MULTIPLE_KEY)) != 0)
goto err;
dbp = NULL;
copied += MULTI_BUFSIZE/SQLITE_DEFAULT_PAGE_SIZE;
}
/*
* Done copying the current table, time to look for a new
* table to copy.
*/
if (ret == DB_NOTFOUND) {
ret = 0;
rc = sqlite3BtreeCloseCursor(&p->destCur);
if (p->srcCur) {
app = p->srcCur->dbp->app_private;
dbp = p->srcCur->dbp;
p->srcCur->close(p->srcCur);
ret = dbp->close(dbp, DB_NOSYNC);
if (app)
sqlite3DbFree(p->pSrcDb, app);
}
p->srcCur = NULL;
if (ret != 0 || rc != SQLITE_OK)
goto err;
p->currentTable += 1;
}
}
goto done;
err: if (ret == SQLITE_DONE)
return ret;
done: return MAP_ERR(rc, ret);
}
int* ROSWOSSegment::lookupByIndex( char* key, int size, Dbc *iter,
u_int32_t flags, void **_ptr, void **buf )
{
size_t retklen, retdlen;
Dbt skey;
void *retkey, *retdata;
bool second_pass = false;
int answer;
memset(&skey, 0, sizeof(skey));
memset(&_bulk_data, 0, sizeof(_bulk_data));
//memset( (*buf), 0, sizeof( (*buf) ) );
//_bulk_data.set_data( _bulk_buffer );
_bulk_data.set_data( (*buf) );
_bulk_data.set_ulen( BUFFER_LENGTH );
_bulk_data.set_flags( DB_DBT_USERMEM );
skey.set_data( key );
skey.set_size( size );
//if ( _debug ) return NULL;
do
{
// if bulk buffer is done, get a new one.
//if ( _p_bulk == NULL )
if ( ((*_ptr)) == NULL )
{
//cout << " Request from iter " << iter << " key " << *(int *)key << " FLAGS? " << flags << endl;
memset( (*buf), 0, sizeof( (*buf) ) );
answer = iter->/*_sort_key_cursor->*/get( &skey, &_bulk_data,
flags | DB_MULTIPLE_KEY );
if ( answer == DB_NOTFOUND )
return NULL;
//DB_MULTIPLE_INIT(_p_bulk, _bulk_data.get_DBT());
DB_MULTIPLE_INIT((*_ptr), _bulk_data.get_DBT());
//cout << " DB_NOTFOUND " << DB_NOTFOUND << " ans " << answer << " -- Getting answers, page size " << BUFFER_LENGTH << " ITER " << iter << " PTR " << (*_ptr) <<endl;
}
//else
//cout << " ROSWOSSegment, *_ptr is not NULL, go go go" << endl;
/*if ( answer == DB_NOTFOUND )
{
//_debug = true;
// THAT really does NOT close the iterator. WHY, I have not
// a faintest idea!
//cout << " ITERATOR " << iter << " IS CLOSED " << endl;
//iter->close();
return NULL;
}*/
//if ( !_p_bulk )
if ( !((*_ptr)) )
return NULL;
// I don't understand that.
//if ( !_p_bulk && second_pass )
//cout << " PTR " << (*_ptr) << " SECOND PASS " << second_pass << endl;
if ( !((*_ptr)) && second_pass )
return NULL;
DB_MULTIPLE_KEY_NEXT((*_ptr), _bulk_data.get_DBT(), retkey, retklen, retdata, retdlen);
//DB_MULTIPLE_KEY_NEXT(_p_bulk, _bulk_data.get_DBT(), retkey, retklen, retdata, retdlen);
// cout << " ANd now, ptr is " << *_ptr << " int ptr " << ((int*)retkey) << endl;
second_pass = true;
}
//while( !_p_bulk ); // if bulk is null, get the buffer.
while( !(*_ptr) ); // if bulk is null, get the buffer.
/*if ( (*((int*)retkey)) > 10500 )
{
cout << DB_NOTFOUND << " <=> " << answer << " Ret answer " << *((int *)retkey) << " retDATA " << *((int *)retdata) << endl;
cout << " Recieved buffer " << buf << " points to buf " << (*buf) << " and ptr " << *_ptr << endl;
*/
/*if ( *((int *)retdata) == 596992 &&
10511 == *((int *)retkey) )
*(int *)retdata = 597221;
if ( *((int *)retdata) == 598272 &&
10524 == *((int *)retkey) )
*(int *)retdata = 598362;*/
//cout << " Finally returning " << ( ((int *)retdata) ) << endl << endl;
return( ((int *)retdata) );
/*
// THIS IS AN ITERATION SEQUENCE. DO NOT ERASE.
while ( _sort_key_cursor->get( &skey, &data, DB_NEXT | DB_MULTIPLE_KEY ) == 0 )
{
for (DB_MULTIPLE_INIT(p, data.get_DBT());;)
{
DB_MULTIPLE_KEY_NEXT(p, data.get_DBT(), retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
//cout << " Ret klen " << *((int *)retkey) << " retdlen " << *((int *)retdata) << endl;
//printf("key: %.*s, data: %.*s\n",
//(int)retklen, (char*)retkey, (int)retdlen, (char*)retdata);
ctr2++;
}
cout << " Got SORT key " << *((int *)skey.get_data()) << endl;
cout << " Curr count " << ctr2 << endl;
}
*/
/*
Dbt dkey, data; // Dbt is a key / data pair
int ret; //,a, b;
char *result = ( char* )malloc( sizeof(int) );
memset(&dkey, 0, sizeof(dkey));
memset(&data, 0, sizeof(data));
data.set_data( &result );
data.set_size( sizeof(int) );
// BDB documentation neglects to mention that once you use your own
// buffer you MUST set this parameter or things will break horribly.
// thanks to a helpful post on a google newsgroup...
data.set_ulen( sizeof(int) );
dkey.set_data( key );
dkey.set_size( size );
try
{
int before = StopWatch::ticks();
ret = iter->get( &dkey, &data, flags );
int after = StopWatch::ticks();
db_access+=(after-before)*StopWatch::secondsPerTick();
//cout << _segNameDebug << " Got " << (after-before)*StopWatch::secondsPerTick() << " for " << db_access << endl;
}
catch(DbException& e)
{
if (DEBUG) cout << "Exception " << e.what() << endl;
// Throw exception here
exit( 1 );
}
if ( ret < 0 )
{
//cout << ret << "Nothing found for key ... " << key << endl;
return NULL;
}
//int a,b;
//memcpy( &a, key, 4 );
//memcpy( &b, data.get_data(), 4 );
//cout << "Receiving key " << b << " for given key " << a << endl;
//if ( fake_key > 599874 )
//return NULL;
//memcpy( result, &fake_key, sizeof(int) );
//fake_key++;
memcpy( result, data.get_data(), sizeof(int) );
return ( int* )result;
*/
}
bool BulkLoad::bulkLoadBody()
{
if (m_cancel)
{
emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkLoad, JobList::Running, job->seq);
NewsGroup* ng = job->ng;
HeaderList* hl = job->headerList;
mphDeletionsList = 0;
sphDeletionsList = 0;
DBC *dbcp = 0;
MultiPartHeader mph;
SinglePartHeader sph;
HeaderBase* hb = 0;
memset(&ckey, 0, sizeof(ckey));
memset(&cdata, 0, sizeof(cdata));
cdata.data = (void *) dataBuffer;
cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
cdata.flags = DB_DBT_USERMEM;
ckey.data = (void *) keyBuffer;
ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
ckey.flags = DB_DBT_USERMEM;
size_t retklen, retdlen;
void *retkey = 0, *retdata = 0;
int ret, t_ret;
void *p = 0;
quint64 count=0;
QTime start = QTime::currentTime();
qDebug() << "Loading started: " << start.toString();
/* Acquire a cursor for the database. */
if ((ret = ng->getDb()->get_DB()->cursor(ng->getDb()->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
{
ng->getDb()->err(ret, "DB->cursor");
return false;
}
quint32 numIgnored = 0;
bool mphFound = false;
for (;;)
{
/*
* Acquire the next set of key/data pairs. This code
* does not handle single key/data pairs that won't fit
* in a BUFFER_LENGTH size buffer, instead returning
* DB_BUFFER_SMALL to our caller.
*/
if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
{
if (ret != DB_NOTFOUND)
ng->getDb()->err(ret, "DBcursor->get");
break;
}
for (DB_MULTIPLE_INIT(p, &cdata);;)
{
DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
if (retdlen){;} // MD TODO compiler .... unused variable
if (*((char *)retdata) == 'm')
{
MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
hb = (HeaderBase*)&mph;
mphFound = true;
}
else if (*((char *)retdata) == 's')
{
SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
hb = (HeaderBase*)&sph;
mphFound = false;
// qDebug() << "Single index = " << sph.getIndex();
}
else
{
// What have we found ?????
qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
continue;
}
if (hb->getStatus() & HeaderBase::MarkedForDeletion)
{
// ignore this header, garbage collection will remove it ...
++numIgnored;
++count; // ... but still count it as bulk delete will reduce the count
if (numIgnored == 1)
{
// These two will be freed by bulk delete
mphDeletionsList = new QList<QString>;
sphDeletionsList = new QList<QString>;
}
if (mphFound)
mphDeletionsList->append(hb->getIndex());
else
sphDeletionsList->append(hb->getIndex());
continue;
}
if (m_cancel)
break;
hl->headerTreeModel->setupTopLevelItem(hb);
++count;
if (count % 50 == 0)
{
QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
" of " + QString("%L1").arg(ng->getTotal()) +
" articles.";
emit progress(job->seq, count, labelText);
QCoreApplication::processEvents();
}
}
if (m_cancel)
break;
}
if ((t_ret = dbcp->close(dbcp)) != 0)
{
ng->getDb()->err(ret, "DBcursor->close");
if (ret == 0)
ret = t_ret;
}
emit loadReady(job->seq);
QString labelText = tr("Loaded ") + QString("%L1").arg(count) +
" of " + QString("%L1").arg(ng->getTotal()) +
" articles.";
emit progress(job->seq, count, labelText);
QCoreApplication::processEvents();
if (!m_cancel)
{
quint64 totalArticles = ng->getTotal();
if (totalArticles != count || ng->unread() > count)
{
qint64 difference = count - totalArticles; // may be negative
quint64 unread = (quint64)qMax((qint64)(ng->unread() + difference), (qint64)0);
if (unread > count)
unread = count;
ng->setTotal(count);
ng->setUnread(unread);
emit updateArticleCounts(ng);
}
}
qDebug() << "Loading finished. Seconds: " << start.secsTo(QTime::currentTime());
qDebug() << "Ignored " << numIgnored << " articles";
qDebug() << "Loaded " << count << " articles";
if (mphDeletionsList) // need bulk deletion
emit addBulkJob(JobList::BulkDelete, ng, hl, mphDeletionsList, sphDeletionsList);
if (m_cancel)
{
emit updateJob(JobList::BulkLoad, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkLoad, JobList::Finished_Ok, job->seq);
return true;
}
static void bulk_get_hash (u_int32_t offset, u_int32_t count, bdb_drv_t *pdrv) {
int ret;
size_t retklen, retdlen;
void *retkey, *retdata;
void* p;
DB* pdb;
DBC* pdbc = NULL;
DBT key, data;
db_recno_t curr = 1;
db_recno_t limit = 1;
db_recno_t actual_count = 0;
ErlDrvTermData *spec;
u_int32_t spec_items, idx;
curr = offset;
limit = offset + count;
pdb = pdrv->pcfg->pdb;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
data.data = pdrv->pcfg->buffer;
data.ulen = pdrv->pcfg->bulk_get_buffer_size_bytes;
data.flags = DB_DBT_USERMEM;
if ((ret = pdb->cursor(pdb, NULL, &pdbc, 0)) != 0) {
return_error_tuple(pdrv, db_strerror(ret));
return;
}
//Fast forward to the correct index
idx = 1;
while ((ret = pdbc->c_get(pdbc, &key, &data, DB_NEXT)) == 0) {
idx++;
if (idx >= offset) { break; }
}
if ((ret != DB_NOTFOUND) && (ret != 0)) {
return_error_tuple(pdrv, db_strerror(ret));
} else if (ret == DB_NOTFOUND) {
return_ok_empty_list(pdrv);
} else {
if ((ret = pdbc->c_get(pdbc, &key, &data, DB_MULTIPLE_KEY | DB_CURRENT)) != 0) {
if (ret == DB_NOTFOUND) {
return_ok_empty_list(pdrv);
} else {
return_error_tuple(pdrv, db_strerror(ret));
}
} else {
//First count the number of recs...
for (DB_MULTIPLE_INIT(p, &data); curr < limit; curr++) {
DB_MULTIPLE_KEY_NEXT(p, &data, retkey, retklen, retdata, retdlen);
if (p == NULL) {
break;
} else {
actual_count++;
}
}
spec_items = (8 * actual_count) + 7;
spec = malloc(sizeof(ErlDrvTermData) * spec_items);
if (spec == NULL) {
return_error_tuple(pdrv, "Could not allocate memory for operation!");
} else {
spec[0] = ERL_DRV_ATOM;
spec[1] = driver_mk_atom("ok");
//Now pipe the data...
p = NULL;
curr = 0;
for (DB_MULTIPLE_INIT(p, &data); curr < actual_count; curr++) {
DB_MULTIPLE_KEY_NEXT(p, &data, retkey, retklen, retdata, retdlen);
if (p == NULL) {
break;
} else {
idx = 2 + (curr * 8);
spec[idx + 0] = ERL_DRV_STRING;
spec[idx + 1] = (ErlDrvTermData)retkey;
spec[idx + 2] = retklen;
spec[idx + 3] = ERL_DRV_STRING;
spec[idx + 4] = (ErlDrvTermData)retdata;
spec[idx + 5] = retdlen;
spec[idx + 6] = ERL_DRV_TUPLE;
spec[idx + 7] = 2;
}
}
spec[spec_items - 5] = ERL_DRV_NIL;
spec[spec_items - 4] = ERL_DRV_LIST;
spec[spec_items - 3] = actual_count + 1;
spec[spec_items - 2] = ERL_DRV_TUPLE;
spec[spec_items - 1] = 2;
#if ((ERL_DRV_EXTENDED_MAJOR_VERSION == 1) || ((ERL_DRV_EXTENDED_MAJOR_VERSION == 2) && (ERL_DRV_EXTENDED_MINOR_VERSION == 0)))
driver_output_term(pdrv->port, spec, spec_items);
#else
ErlDrvTermData mkport = driver_mk_port(pdrv->port);
erl_drv_output_term(mkport, spec, spec_items);
#endif
free(spec);
}
}
}
pdbc->c_close(pdbc);
return;
}
int puavo_conf_db_get_all(struct puavo_conf *const conf,
struct puavo_conf_list *const list,
struct puavo_conf_err *const errp)
{
DBC *db_cursor = NULL;
DBT db_null;
DBT db_batch;
size_t length = 0;
char **keys = NULL;
char **values = NULL;
int ret = -1;
int db_error;
memset(&db_null, 0, sizeof(DBT));
memset(&db_batch, 0, sizeof(DBT));
db_batch.flags = DB_DBT_USERMEM;
db_batch.ulen = PUAVO_CONF_DEFAULT_DB_BATCH_SIZE;
db_batch.data = malloc(db_batch.ulen);
if (!db_batch.data) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS, 0,
"Failed to get all parameters");
goto out;
}
db_error = conf->db->cursor(conf->db, NULL, &db_cursor, 0);
if (db_error) {
db_cursor = NULL;
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
goto out;
}
/* Iterate key/value pairs in batches until all are found. */
while (1) {
void *batch_iterator;
/* Get the next batch of key-value pairs. */
db_error = db_cursor->get(db_cursor, &db_null, &db_batch,
DB_MULTIPLE_KEY | DB_NEXT);
switch (db_error) {
case 0:
break;
case DB_NOTFOUND:
ret = 0;
goto out;
default:
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
goto out;
}
/* Iterate the batch. */
DB_MULTIPLE_INIT(batch_iterator, &db_batch);
while (1) {
char *key;
char *val;
size_t key_size;
size_t val_size;
char **new_keys;
char **new_values;
DB_MULTIPLE_KEY_NEXT(batch_iterator, &db_batch,
key, key_size, val, val_size);
if (!batch_iterator)
break; /* The batch is empty. */
new_keys = realloc(keys,
sizeof(char *) * (length + 1));
if (!new_keys) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS,
0, "Failed to get all "
"parameters");
goto out;
}
keys = new_keys;
new_values = realloc(values,
sizeof(char *) * (length + 1));
if (!new_values) {
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_SYS,
0, "Failed to get all "
"parameters");
goto out;
}
values = new_values;
++length;
keys[length - 1] = strndup(key, key_size);
values[length - 1] = strndup(val, val_size);
}
}
ret = 0;
out:
if (db_cursor) {
db_error = db_cursor->close(db_cursor);
/* Obey exit-on-first-error policy: Do not shadow any
* existing error, record close error only if we are
* cleaning up without any earlier errors. */
if (!ret && db_error) {
ret = -1;
puavo_conf_err_set(errp, PUAVO_CONF_ERRNUM_DB, db_error,
"Failed to get all parameters");
}
}
if (ret) {
size_t i;
for (i = 0; i < length; ++i) {
free(keys[i]);
free(values[i]);
}
free(keys);
free(values);
} else {
list->keys = keys;
list->values = values;
list->length = length;
}
free(db_batch.data);
return ret;
}
