void BtRecExample::run() { db_recno_t recno; int ret; char buf[1024]; // Acquire a cursor for the database. dbp->cursor(NULL, &dbcp, 0); // // Prompt the user for a record number, then retrieve and display // that record. // for (;;) { // Get a record number. cout << "recno #> "; cout.flush(); if (fgets(buf, sizeof(buf), stdin) == NULL) break; recno = atoi(buf); // // Start with a fresh key each time, // the dbp->get() routine returns // the key and data pair, not just the key! // Dbt key(&recno, sizeof(recno)); Dbt data; if ((ret = dbcp->get(&key, &data, DB_SET_RECNO)) != 0) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } // Display the key and data. show("k/d\t", &key, &data); // Move the cursor a record forward. if ((ret = dbcp->get(&key, &data, DB_NEXT)) != 0) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } // Display the key and data. show("next\t", &key, &data); // // Retrieve the record number for the following record into // local memory. // data.set_data(&recno); data.set_size(sizeof(recno)); data.set_ulen(sizeof(recno)); data.set_flags(data.get_flags() | DB_DBT_USERMEM); if ((ret = dbcp->get(&key, &data, DB_GET_RECNO)) != 0) { if (ret != DB_NOTFOUND && ret != DB_KEYEMPTY) { dbp->err(ret, "DBcursor->get"); throw DbException(ret); } } else { cout << "retrieved recno: " << (u_long)recno << "\n"; } } dbcp->close(); dbcp = NULL; }
bool BulkHeaderGroup::BulkHeaderGroupBody() { // belt and braces key.set_data(keymem); data.set_data(datamem); if (m_cancel) { emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq); return false; } emit updateJob(JobList::BulkHeaderGroup, JobList::Running, job->seq); NewsGroup* ng = job->ng; Db* db = ng->getDb(); MultiPartHeader mph; SinglePartHeader sph; HeaderBase* hb = 0; HeaderGroup* headerGroup = 0; HeaderGroup* advancedHeaderGroup = 0; // typedef QMap<QString, QString> HeaderGroupIndexes; // subj, headerGroup index // typedef QMap<QString, HeaderGroup*> HeaderGroups; // headerGroup index, headerGroup * HeaderGroupIndexes headerGroupIndexes; HeaderGroups headerGroups; DBC *dbcp = 0; DBT ckey, cdata; memset(&ckey, 0, sizeof(ckey)); memset(&cdata, 0, sizeof(cdata)); size_t retklen, retdlen; void *retkey = 0, *retdata = 0; int ret, t_ret; void *p = 0; quint64 count=0; cdata.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH]; cdata.ulen = HEADER_BULK_BUFFER_LENGTH; cdata.flags = DB_DBT_USERMEM; ckey.data = (void *) new char[HEADER_BULK_BUFFER_LENGTH]; ckey.ulen = HEADER_BULK_BUFFER_LENGTH; ckey.flags = DB_DBT_USERMEM; /* Acquire a cursor for the database. */ if ((ret = db->get_DB()->cursor(db->get_DB(), NULL, &dbcp, DB_CURSOR_BULK)) != 0) { db->err(ret, "DB->cursor"); char* ptr = 0; ptr = (char*)(ckey.data); Q_DELETE_ARRAY(ptr); ptr = (char*)(cdata.data); Q_DELETE_ARRAY(ptr); return false; } // To save the group records ng->articlesNeedDeleting(false); // Store the data in the database - flush first ... u_int32_t delCount; uchar keymem[KEYMEM_SIZE]; uchar datamem[DATAMEM_SIZE]; Dbt key, data; char* p2 = 0; QByteArray ba; const char *k = 0; key.set_flags(DB_DBT_USERMEM); key.set_data(&keymem); key.set_ulen(KEYMEM_SIZE); data.set_flags(DB_DBT_USERMEM); data.set_ulen(DATAMEM_SIZE); data.set_data(&datamem); QString subj = "MDQuban", from = "MDQuban"; //QString rs1 = "^(.*)(\".*\")"; //QString rs2 = "^(.*)\\s-\\s(.*)$"; //QString rs3 = "^(\\S+.*)\\[.*\\].*(\".*\")"; //QString rs3 = "^(.*)\\s-\\s.*\\s-\\s(.*)$"; QRegExp rx[3]; bool rxPosBack[3]; bool noRegexpGrouping; QString recKey, storeIndex; QString prevSubj = "MDQuban", prevFrom = "MDQuban"; int pos; bool newGroup = false; bool mphFound = false; quint32 grouped = 0, single = 0, numGroups = 0; qint16 stringDiff = -1; bool prevGroup = false; bool advancedPlacement = false; bool skipAdvanced = false; noRegexpGrouping = ng->isThereNoRegexOnGrouping(); if (noRegexpGrouping == false) // need regex for grouping { rx[0].setPattern(ng->getGroupRE1()); rx[1].setPattern(ng->getGroupRE2()); rx[2].setPattern(ng->getGroupRE3()); rxPosBack[0] = ng->getGroupRE1Back(); rxPosBack[1] = ng->getGroupRE2Back(); rxPosBack[2] = ng->getGroupRE3Back(); } ng->getGroupingDb()->truncate(0, &delCount, 0); qDebug() << "Deleted " << delCount << " records from group db"; QMapIterator<QString, QString> it(headerGroupIndexes); QString advancedIndex; for (;;) { /* * Acquire the next set of key/data pairs. This code * does not handle single key/data pairs that won't fit * in a BUFFER_LENGTH size buffer, instead returning * DB_BUFFER_SMALL to our caller. */ if ((ret = dbcp->get(dbcp, &ckey, &cdata, DB_MULTIPLE_KEY | DB_NEXT)) != 0) { if (ret != DB_NOTFOUND) db->err(ret, "DBcursor->get"); break; } for (DB_MULTIPLE_INIT(p, &cdata);;) { DB_MULTIPLE_KEY_NEXT(p, &cdata, retkey, retklen, retdata, retdlen); if (p == NULL) break; if (retdlen){;} // MD TODO compiler .... unused variable recKey = QString::fromLocal8Bit((char*)retkey, retklen); if (*((char *)retdata) == 'm') { MultiPartHeader::getMultiPartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &mph); hb = (HeaderBase*)&mph; mphFound = true; } else if (*((char *)retdata) == 's') { SinglePartHeader::getSinglePartHeader((unsigned int)retklen, (char *)retkey, (char *)retdata, &sph); hb = (HeaderBase*)&sph; mphFound = false; } else { // What have we found ????? qDebug() << "Found unexpected identifier for header : " << (char)*((char *)retdata); continue; } ++count; prevSubj = subj; prevFrom = from; subj = hb->getSubj(); from = hb->getFrom(); if (noRegexpGrouping == false) // need regex for grouping { for (int i=0; i<3; ++i) { if (rx[i].isEmpty() == false) { if (rxPosBack[i] == true) // from the back { pos = subj.lastIndexOf(rx[i]); if (pos != -1) subj.truncate(pos); } else // from the front { pos = rx[i].indexIn(subj); if (pos > -1) subj = rx[i].cap(0); } } } } //qDebug() << "Stripped down to: " << subj; stringDiff = -1; if (prevFrom != from) // change of contributor { newGroup = true; } else // same contributor { if ((stringDiff = levenshteinDistance(prevSubj, subj)) > ng->getMatchDistance()) // no match ... newGroup = true; else newGroup = false; //qDebug() << "Diff between " << prevSubj << " and " << subj << " is " << stringDiff; } if (newGroup) { if (ng->isThereAdvancedGrouping()) { it.toFront(); // decide if we can match to a previous group while (it.hasNext()) { it.next(); if ((stringDiff = levenshteinDistance(it.key(), subj)) <= ng->getMatchDistance()) // match ... { // The index for this group is in it.value() // See if we have the HeaderGroup in our cache headerGroups) if (headerGroups.contains(it.value())) { advancedHeaderGroup = headerGroups.value(it.value()); } else // not in cache { advancedIndex = it.value(); advancedHeaderGroup = getGroup(ng, advancedIndex); if (advancedHeaderGroup) { headerGroups.insert(advancedIndex, advancedHeaderGroup); } else // db read failed .. { skipAdvanced = true; } } if (skipAdvanced == false) { if (mphFound) advancedHeaderGroup->addMphKey(recKey); else advancedHeaderGroup->addSphKey(recKey); advancedPlacement = true; subj = prevSubj; // ignore this header as it's been placed out of sequence from = prevFrom; newGroup = false; // as we managed to relocate to an existing group break; // stop looking at previous groups } else skipAdvanced = false; } } } } if (newGroup) { if (prevGroup) // save before moving on { ba = storeIndex.toLocal8Bit(); k= ba.constData(); memcpy(keymem, k, storeIndex.length()); key.set_size(storeIndex.length()); p2=headerGroup->data(); data.set_data(p2); data.set_size(headerGroup->getRecordSize()); ret=ng->getGroupingDb()->put(NULL, &key, &data, 0); if (ret!=0) qDebug("Error updating record: %d", ret); if (ng->isThereAdvancedGrouping()) headerGroupIndexes.insert(storeIndex.section('\n', 0, 0), storeIndex); Q_DELETE_ARRAY(p2); Q_DELETE(headerGroup); numGroups++; } prevGroup = true; storeIndex = subj % "\n" % from; headerGroup = new HeaderGroup(); headerGroup->setDisplayName(subj); headerGroup->setPostingDate(hb->getPostingDate()); headerGroup->setDownloadDate(hb->getDownloadDate()); headerGroup->setStatus(hb->getStatus()); headerGroup->setNextDistance(stringDiff); } // if we've found somewhere else to place this header then don't add again if (!advancedPlacement) { if (mphFound) headerGroup->addMphKey(recKey); else headerGroup->addSphKey(recKey); } else advancedPlacement = false; if (count % 250 == 0) { QCoreApplication::processEvents(); emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " + QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq); } if (m_cancel) { emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq); return false; } } if (m_cancel) { emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq); return false; } } if ((t_ret = dbcp->close(dbcp)) != 0) { db->err(ret, "DBcursor->close"); if (ret == 0) ret = t_ret; } char* ptr = ((char*)ckey.data); Q_DELETE_ARRAY(ptr); ptr = ((char*)cdata.data); Q_DELETE_ARRAY(ptr); if (headerGroups.count()) { qDeleteAll(headerGroups); headerGroups.clear(); } qDebug() << "Multi = " << grouped << ", single = " << single; ng->setHeadersNeedGrouping(false); // Finally update the newsgroup emit saveGroup(ng); emit updateJob(JobList::BulkHeaderGroup, tr("Header bulk grouping for newsgroup ") + job->ng->getAlias() + ": " + QString::number(count) + " out of " + QString::number(ng->getTotal()) + tr(" grouped"), job->seq); if (m_cancel) { emit updateJob(JobList::BulkHeaderGroup, JobList::Cancelled, job->seq); return false; } emit logEvent(tr("Bulk grouping of ") + ng->getTotal() + tr(" articles completed successfully.")); emit updateJob(JobList::BulkHeaderGroup, JobList::Finished_Ok, job->seq); ng->setTotalGroups(numGroups); Q_DELETE(headerGroup); return true; }
int main() { u_int32_t env_flags = DB_CREATE | // If the environment does not exist, create it. DB_INIT_MPOOL; // Initialize the in-memory cache. //std::string envHome("./"); std::string envHome("/home/kirill"); u_int32_t db_flags = DB_RDONLY; // only read std::string dbName("X-po2s.db"); //std::string dbName("X-so2p.db"); //std::string dbName("X-sp2o.db"); DbEnv myEnv(0); Db *myDb; // Instantiate the Db object Dbc *cursorp; // cursor try { cout << "X-po2s.db output:" << endl; myEnv.open(envHome.c_str(), env_flags, 0); myDb = new Db(&myEnv, 0); myDb->open(NULL, dbName.c_str(), NULL, DB_BTREE, db_flags, 0); // Get a cursor myDb->cursor(NULL, &cursorp, 0); Dbt key, data; // Position the cursor to the first record in the database whose // key and data begin with the correct strings. int ret = cursorp->get(&key, &data, DB_NEXT); while (ret != DB_NOTFOUND) { cout << "..." << endl; std::cout << "key: " << (char *)key.get_data() << "data: " << (char *)data.get_data()<< std::endl; ret = cursorp->get(&key, &data, DB_NEXT); } // Cursors must be closed if (cursorp != NULL) cursorp->close(); if (myDb != NULL) { myDb->close(0); } myEnv.close(0); cout << "Closing ..." << endl; } // Must catch both DbException and std::exception catch(DbException &e) { myDb->err(e.get_errno(), "Database open failed %s", dbName.c_str()); throw e; } catch(std::exception &e) { // No DB error number available, so use errx myDb->errx("Error opening database: %s", e.what()); throw e; } return 0; }
int RepMgr::doloop() { Db *dbp; Dbt key, data; char buf[BUFSIZE], *rbuf; int ret; dbp = NULL; memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); ret = 0; for (;;) { if (dbp == NULL) { dbp = new Db(&dbenv, 0); try { dbp->open(NULL, DATABASE, NULL, DB_BTREE, DB_CREATE | DB_AUTO_COMMIT, 0); } catch(DbException dbe) { dbenv.err(ret, "DB->open"); throw dbe; } } cout << "QUOTESERVER" ; cout << "> " << flush; if (fgets(buf, sizeof(buf), stdin) == NULL) break; if (strtok(&buf[0], " \t\n") == NULL) { switch ((ret = print_stocks(dbp))) { case 0: continue; default: dbp->err(ret, "Error traversing data"); goto err; } } rbuf = strtok(NULL, " \t\n"); if (rbuf == NULL || rbuf[0] == '\0') { if (strncmp(buf, "exit", 4) == 0 || strncmp(buf, "quit", 4) == 0) break; dbenv.errx("Format: TICKER VALUE"); continue; } key.set_data(buf); key.set_size((u_int32_t)strlen(buf)); data.set_data(rbuf); data.set_size((u_int32_t)strlen(rbuf)); if ((ret = dbp->put(NULL, &key, &data, 0)) != 0) { dbp->err(ret, "DB->put"); if (ret != DB_KEYEXIST) goto err; } } err: if (dbp != NULL) (void)dbp->close(DB_NOSYNC); return (ret); }
int RepMgrGSG::doloop() { Dbt key, data; Db *dbp; char buf[BUFSIZE], *rbuf; int ret; dbp = 0; memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); ret = 0; for (;;) { if (dbp == 0) { dbp = new Db(&dbenv, 0); try { dbp->open(NULL, DATABASE, NULL, DB_BTREE, app_data.is_master ? DB_CREATE | DB_AUTO_COMMIT : DB_AUTO_COMMIT, 0); } catch(DbException dbe) { // It is expected that this condition will be triggered // when client sites start up. It can take a while for // the master site to be found and synced, and no DB will // be available until then. if (dbe.get_errno() == ENOENT) { cout << "No stock db available yet - retrying." << endl; try { dbp->close(0); } catch (DbException dbe2) { cout << "Unexpected error closing after failed" << " open, message: " << dbe2.what() << endl; dbp = NULL; goto err; } dbp = NULL; sleep(SLEEPTIME); continue; } else { dbenv.err(ret, "DB->open"); throw dbe; } } } cout << "QUOTESERVER" ; if (!app_data.is_master) cout << "(read-only)"; cout << "> " << flush; if (fgets(buf, sizeof(buf), stdin) == NULL) break; if (strtok(&buf[0], " \t\n") == NULL) { switch ((ret = print_stocks(dbp))) { case 0: continue; case DB_REP_HANDLE_DEAD: (void)dbp->close(DB_NOSYNC); cout << "closing db handle due to rep handle dead" << endl; dbp = NULL; continue; default: dbp->err(ret, "Error traversing data"); goto err; } } rbuf = strtok(NULL, " \t\n"); if (rbuf == NULL || rbuf[0] == '\0') { if (strncmp(buf, "exit", 4) == 0 || strncmp(buf, "quit", 4) == 0) break; dbenv.errx("Format: TICKER VALUE"); continue; } if (!app_data.is_master) { dbenv.errx("Can't update at client"); continue; } key.set_data(buf); key.set_size((u_int32_t)strlen(buf)); data.set_data(rbuf); data.set_size((u_int32_t)strlen(rbuf)); if ((ret = dbp->put(NULL, &key, &data, 0)) != 0) { dbp->err(ret, "DB->put"); if (ret != DB_KEYEXIST) goto err; } } err: if (dbp != 0) { (void)dbp->close(DB_NOSYNC); } return (ret); }