void BtRecExample::run()
{
db_recno_t recno;
int ret;
char buf[1024];
// Acquire a cursor for the database.
dbp->cursor(NULL, &dbcp, 0);
//
// Prompt the user for a record number, then retrieve and display
// that record.
//
for (;;) {
// Get a record number.
cout << "recno #> ";
cout.flush();
if (fgets(buf, sizeof(buf), stdin) == NULL)
break;
recno = atoi(buf);
//
// Start with a fresh key each time,
// the dbp->get() routine returns
// the key and data pair, not just the key!
//
Dbt key(&recno, sizeof(recno));
Dbt data;
if ((ret = dbcp->get(&key, &data, DB_SET_RECNO)) != 0) {
dbp->err(ret, "DBcursor->get");
throw DbException(ret);
}
// Display the key and data.
show("k/d\t", &key, &data);
// Move the cursor a record forward.
if ((ret = dbcp->get(&key, &data, DB_NEXT)) != 0) {
dbp->err(ret, "DBcursor->get");
throw DbException(ret);
}
// Display the key and data.
show("next\t", &key, &data);
//
// Retrieve the record number for the following record into
// local memory.
//
data.set_data(&recno);
data.set_size(sizeof(recno));
data.set_ulen(sizeof(recno));
data.set_flags(data.get_flags() | DB_DBT_USERMEM);
if ((ret = dbcp->get(&key, &data, DB_GET_RECNO)) != 0) {
if (ret != DB_NOTFOUND && ret != DB_KEYEMPTY) {
dbp->err(ret, "DBcursor->get");
throw DbException(ret);
}
}
else {
cout << "retrieved recno: " << (u_long)recno << "\n";
}
}
dbcp->close();
dbcp = NULL;
}
bool BulkHeaderGroup::BulkHeaderGroupBody()
{
// belt and braces
key.set_data(keymem);
data.set_data(datamem);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit updateJob(JobList::BulkHeaderGroup, JobList::Running, job->seq);
NewsGroup* ng = job->ng;
Db* db = ng->getDb();
MultiPartHeader mph;
SinglePartHeader sph;
HeaderBase* hb = 0;
HeaderGroup* headerGroup = 0;
HeaderGroup* advancedHeaderGroup = 0;
// typedef QMap<QString, QString> HeaderGroupIndexes; // subj, headerGroup index
// typedef QMap<QString, HeaderGroup*> HeaderGroups; // headerGroup index, headerGroup *
HeaderGroupIndexes headerGroupIndexes;
HeaderGroups headerGroups;
DBC *dbcp = 0;
DBT ckey, cdata;
memset(&ckey, 0, sizeof(ckey));
memset(&cdata, 0, sizeof(cdata));
size_t retklen, retdlen;
void *retkey = 0, *retdata = 0;
int ret, t_ret;
void *p = 0;
quint64 count=0;
cdata.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
cdata.ulen = HEADER_BULK_BUFFER_LENGTH;
cdata.flags = DB_DBT_USERMEM;
ckey.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH];
ckey.ulen = HEADER_BULK_BUFFER_LENGTH;
ckey.flags = DB_DBT_USERMEM;
/* Acquire a cursor for the database. */
if ((ret = db->get_DB()->cursor(db->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0)
{
db->err(ret, "DB->cursor");
char* ptr = 0;
ptr = (char*)(ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = (char*)(cdata.data);
Q_DELETE_ARRAY(ptr);
return false;
}
// To save the group records
ng->articlesNeedDeleting(false);
// Store the data in the database - flush first ...
u_int32_t delCount;
uchar keymem[KEYMEM_SIZE];
uchar datamem[DATAMEM_SIZE];
Dbt key, data;
char* p2 = 0;
QByteArray ba;
const char *k = 0;
key.set_flags(DB_DBT_USERMEM);
key.set_data(&keymem);
key.set_ulen(KEYMEM_SIZE);
data.set_flags(DB_DBT_USERMEM);
data.set_ulen(DATAMEM_SIZE);
data.set_data(&datamem);
QString subj = "MDQuban", from = "MDQuban";
//QString rs1 = "^(.*)(\".*\")";
//QString rs2 = "^(.*)\\s-\\s(.*)$";
//QString rs3 = "^(\\S+.*)\\[.*\\].*(\".*\")";
//QString rs3 = "^(.*)\\s-\\s.*\\s-\\s(.*)$";
QRegExp rx[3];
bool rxPosBack[3];
bool noRegexpGrouping;
QString recKey, storeIndex;
QString prevSubj = "MDQuban", prevFrom = "MDQuban";
int pos;
bool newGroup = false;
bool mphFound = false;
quint32 grouped = 0,
single = 0,
numGroups = 0;
qint16 stringDiff = -1;
bool prevGroup = false;
bool advancedPlacement = false;
bool skipAdvanced = false;
noRegexpGrouping = ng->isThereNoRegexOnGrouping();
if (noRegexpGrouping == false) // need regex for grouping
{
rx[0].setPattern(ng->getGroupRE1());
rx[1].setPattern(ng->getGroupRE2());
rx[2].setPattern(ng->getGroupRE3());
rxPosBack[0] = ng->getGroupRE1Back();
rxPosBack[1] = ng->getGroupRE2Back();
rxPosBack[2] = ng->getGroupRE3Back();
}
ng->getGroupingDb()->truncate(0, &delCount, 0);
qDebug() << "Deleted " << delCount << " records from group db";
QMapIterator<QString, QString> it(headerGroupIndexes);
QString advancedIndex;
for (;;)
{
/*
* Acquire the next set of key/data pairs. This code
* does not handle single key/data pairs that won't fit
* in a BUFFER_LENGTH size buffer, instead returning
* DB_BUFFER_SMALL to our caller.
*/
if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0)
{
if (ret != DB_NOTFOUND)
db->err(ret, "DBcursor->get");
break;
}
for (DB_MULTIPLE_INIT(p, &cdata);;)
{
DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen);
if (p == NULL)
break;
if (retdlen){;} // MD TODO compiler .... unused variable
recKey = QString::fromLocal8Bit((char*)retkey, retklen);
if (*((char *)retdata) == 'm')
{
MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph);
hb = (HeaderBase*)&mph;
mphFound = true;
}
else if (*((char *)retdata) == 's')
{
SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph);
hb = (HeaderBase*)&sph;
mphFound = false;
}
else
{
// What have we found ?????
qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata);
continue;
}
++count;
prevSubj = subj;
prevFrom = from;
subj = hb->getSubj();
from = hb->getFrom();
if (noRegexpGrouping == false) // need regex for grouping
{
for (int i=0; i<3; ++i)
{
if (rx[i].isEmpty() == false)
{
if (rxPosBack[i] == true) // from the back
{
pos = subj.lastIndexOf(rx[i]);
if (pos != -1)
subj.truncate(pos);
}
else // from the front
{
pos = rx[i].indexIn(subj);
if (pos > -1)
subj = rx[i].cap(0);
}
}
}
}
//qDebug() << "Stripped down to: " << subj;
stringDiff = -1;
if (prevFrom != from) // change of contributor
{
newGroup = true;
}
else // same contributor
{
if ((stringDiff = levenshteinDistance(prevSubj, subj)) > ng->getMatchDistance()) // no match ...
newGroup = true;
else
newGroup = false;
//qDebug() << "Diff between " << prevSubj << " and " << subj << " is " << stringDiff;
}
if (newGroup)
{
if (ng->isThereAdvancedGrouping())
{
it.toFront();
// decide if we can match to a previous group
while (it.hasNext())
{
it.next();
if ((stringDiff = levenshteinDistance(it.key(), subj)) <= ng->getMatchDistance()) // match ...
{
// The index for this group is in it.value()
// See if we have the HeaderGroup in our cache headerGroups)
if (headerGroups.contains(it.value()))
{
advancedHeaderGroup = headerGroups.value(it.value());
}
else // not in cache
{
advancedIndex = it.value();
advancedHeaderGroup = getGroup(ng, advancedIndex);
if (advancedHeaderGroup)
{
headerGroups.insert(advancedIndex, advancedHeaderGroup);
}
else // db read failed ..
{
skipAdvanced = true;
}
}
if (skipAdvanced == false)
{
if (mphFound)
advancedHeaderGroup->addMphKey(recKey);
else
advancedHeaderGroup->addSphKey(recKey);
advancedPlacement = true;
subj = prevSubj; // ignore this header as it's been placed out of sequence
from = prevFrom;
newGroup = false; // as we managed to relocate to an existing group
break; // stop looking at previous groups
}
else
skipAdvanced = false;
}
}
}
}
if (newGroup)
{
if (prevGroup) // save before moving on
{
ba = storeIndex.toLocal8Bit();
k= ba.constData();
memcpy(keymem, k, storeIndex.length());
key.set_size(storeIndex.length());
p2=headerGroup->data();
data.set_data(p2);
data.set_size(headerGroup->getRecordSize());
ret=ng->getGroupingDb()->put(NULL, &key, &data, 0);
if (ret!=0)
qDebug("Error updating record: %d", ret);
if (ng->isThereAdvancedGrouping())
headerGroupIndexes.insert(storeIndex.section('\n', 0, 0), storeIndex);
Q_DELETE_ARRAY(p2);
Q_DELETE(headerGroup);
numGroups++;
}
prevGroup = true;
storeIndex = subj % "\n" % from;
headerGroup = new HeaderGroup();
headerGroup->setDisplayName(subj);
headerGroup->setPostingDate(hb->getPostingDate());
headerGroup->setDownloadDate(hb->getDownloadDate());
headerGroup->setStatus(hb->getStatus());
headerGroup->setNextDistance(stringDiff);
}
// if we've found somewhere else to place this header then don't add again
if (!advancedPlacement)
{
if (mphFound)
headerGroup->addMphKey(recKey);
else
headerGroup->addSphKey(recKey);
}
else
advancedPlacement = false;
if (count % 250 == 0)
{
QCoreApplication::processEvents();
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
}
if ((t_ret = dbcp->close(dbcp)) != 0)
{
db->err(ret, "DBcursor->close");
if (ret == 0)
ret = t_ret;
}
char* ptr = ((char*)ckey.data);
Q_DELETE_ARRAY(ptr);
ptr = ((char*)cdata.data);
Q_DELETE_ARRAY(ptr);
if (headerGroups.count())
{
qDeleteAll(headerGroups);
headerGroups.clear();
}
qDebug() << "Multi = " << grouped << ", single = " << single;
ng->setHeadersNeedGrouping(false);
// Finally update the newsgroup
emit saveGroup(ng);
emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " +
QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq);
if (m_cancel)
{
emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq);
return false;
}
emit logEvent(tr("Bulk grouping of ") + ng->getTotal() + tr(" articles completed successfully."));
emit updateJob(JobList::BulkHeaderGroup, JobList::Finished_Ok, job->seq);
ng->setTotalGroups(numGroups);
Q_DELETE(headerGroup);
return true;
}
int main()
{
u_int32_t env_flags = DB_CREATE | // If the environment does not exist, create it.
DB_INIT_MPOOL; // Initialize the in-memory cache.
//std::string envHome("./");
std::string envHome("/home/kirill");
u_int32_t db_flags = DB_RDONLY; // only read
std::string dbName("X-po2s.db");
//std::string dbName("X-so2p.db");
//std::string dbName("X-sp2o.db");
DbEnv myEnv(0);
Db *myDb; // Instantiate the Db object
Dbc *cursorp; // cursor
try
{
cout << "X-po2s.db output:" << endl;
myEnv.open(envHome.c_str(), env_flags, 0);
myDb = new Db(&myEnv, 0);
myDb->open(NULL, dbName.c_str(), NULL, DB_BTREE, db_flags, 0);
// Get a cursor
myDb->cursor(NULL, &cursorp, 0);
Dbt key, data;
// Position the cursor to the first record in the database whose
// key and data begin with the correct strings.
int ret = cursorp->get(&key, &data, DB_NEXT);
while (ret != DB_NOTFOUND)
{
cout << "..." << endl;
std::cout << "key: " << (char *)key.get_data()
<< "data: " << (char *)data.get_data()<< std::endl;
ret = cursorp->get(&key, &data, DB_NEXT);
}
// Cursors must be closed
if (cursorp != NULL)
cursorp->close();
if (myDb != NULL) {
myDb->close(0);
}
myEnv.close(0);
cout << "Closing ..." << endl;
}
// Must catch both DbException and std::exception
catch(DbException &e)
{
myDb->err(e.get_errno(), "Database open failed %s", dbName.c_str());
throw e;
}
catch(std::exception &e)
{
// No DB error number available, so use errx
myDb->errx("Error opening database: %s", e.what());
throw e;
}
return 0;
}
int RepMgr::doloop()
{
Db *dbp;
Dbt key, data;
char buf[BUFSIZE], *rbuf;
int ret;
dbp = NULL;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
ret = 0;
for (;;) {
if (dbp == NULL) {
dbp = new Db(&dbenv, 0);
try {
dbp->open(NULL, DATABASE, NULL, DB_BTREE,
DB_CREATE | DB_AUTO_COMMIT, 0);
} catch(DbException dbe) {
dbenv.err(ret, "DB->open");
throw dbe;
}
}
cout << "QUOTESERVER" ;
cout << "> " << flush;
if (fgets(buf, sizeof(buf), stdin) == NULL)
break;
if (strtok(&buf[0], " \t\n") == NULL) {
switch ((ret = print_stocks(dbp))) {
case 0:
continue;
default:
dbp->err(ret, "Error traversing data");
goto err;
}
}
rbuf = strtok(NULL, " \t\n");
if (rbuf == NULL || rbuf[0] == '\0') {
if (strncmp(buf, "exit", 4) == 0 ||
strncmp(buf, "quit", 4) == 0)
break;
dbenv.errx("Format: TICKER VALUE");
continue;
}
key.set_data(buf);
key.set_size((u_int32_t)strlen(buf));
data.set_data(rbuf);
data.set_size((u_int32_t)strlen(rbuf));
if ((ret = dbp->put(NULL, &key, &data, 0)) != 0)
{
dbp->err(ret, "DB->put");
if (ret != DB_KEYEXIST)
goto err;
}
}
err:
if (dbp != NULL)
(void)dbp->close(DB_NOSYNC);
return (ret);
}
int RepMgrGSG::doloop()
{
Dbt key, data;
Db *dbp;
char buf[BUFSIZE], *rbuf;
int ret;
dbp = 0;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
ret = 0;
for (;;) {
if (dbp == 0) {
dbp = new Db(&dbenv, 0);
try {
dbp->open(NULL, DATABASE, NULL, DB_BTREE,
app_data.is_master ? DB_CREATE | DB_AUTO_COMMIT :
DB_AUTO_COMMIT, 0);
} catch(DbException dbe) {
// It is expected that this condition will be triggered
// when client sites start up. It can take a while for
// the master site to be found and synced, and no DB will
// be available until then.
if (dbe.get_errno() == ENOENT) {
cout << "No stock db available yet - retrying." << endl;
try {
dbp->close(0);
} catch (DbException dbe2) {
cout << "Unexpected error closing after failed" <<
" open, message: " << dbe2.what() << endl;
dbp = NULL;
goto err;
}
dbp = NULL;
sleep(SLEEPTIME);
continue;
} else {
dbenv.err(ret, "DB->open");
throw dbe;
}
}
}
cout << "QUOTESERVER" ;
if (!app_data.is_master)
cout << "(read-only)";
cout << "> " << flush;
if (fgets(buf, sizeof(buf), stdin) == NULL)
break;
if (strtok(&buf[0], " \t\n") == NULL) {
switch ((ret = print_stocks(dbp))) {
case 0:
continue;
case DB_REP_HANDLE_DEAD:
(void)dbp->close(DB_NOSYNC);
cout << "closing db handle due to rep handle dead" << endl;
dbp = NULL;
continue;
default:
dbp->err(ret, "Error traversing data");
goto err;
}
}
rbuf = strtok(NULL, " \t\n");
if (rbuf == NULL || rbuf[0] == '\0') {
if (strncmp(buf, "exit", 4) == 0 ||
strncmp(buf, "quit", 4) == 0)
break;
dbenv.errx("Format: TICKER VALUE");
continue;
}
if (!app_data.is_master) {
dbenv.errx("Can't update at client");
continue;
}
key.set_data(buf);
key.set_size((u_int32_t)strlen(buf));
data.set_data(rbuf);
data.set_size((u_int32_t)strlen(rbuf));
if ((ret = dbp->put(NULL, &key, &data, 0)) != 0)
{
dbp->err(ret, "DB->put");
if (ret != DB_KEYEXIST)
goto err;
}
}
err: if (dbp != 0) {
(void)dbp->close(DB_NOSYNC);
}
return (ret);
}
