HeaderBase* HeaderReadWorker::dbBinHeaderGet(QString index) { Dbt key, data; memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); data.set_flags(DB_DBT_MALLOC); HeaderBase* hb = 0; MultiPartHeader *mph = 0; SinglePartHeader *sph = 0; char* dataBlock = 0; QByteArray ba = index.toLocal8Bit(); const char *indexCharArr = ba.constData(); key.set_data((void*) indexCharArr); key.set_size(index.length()); if ((db->get(NULL, &key, &data, 0)) == 0) { dataBlock = (char*)data.get_data(); if (*dataBlock == 'm') { mph = new MultiPartHeader(key.get_size(), (char*)key.get_data(), dataBlock); //qDebug() << "Just read mph with key " << mph->multiPartKey; hb = (HeaderBase*)mph; } else if (*dataBlock == 's') { sph = new SinglePartHeader(key.get_size(), (char*)key.get_data(), dataBlock); hb = (HeaderBase*)sph; } Q_FREE(dataBlock); } return hb; }
HeaderGroup* BulkHeaderGroup::getGroup(NewsGroup* ng, QString& articleIndex) { HeaderGroup *hg = 0; int ret; Dbt groupkey; Dbt groupdata; memset(&groupkey, 0, sizeof(groupkey)); memset(&groupdata, 0, sizeof(groupdata)); groupdata.set_flags(DB_DBT_MALLOC); QByteArray ba = articleIndex.toLocal8Bit(); const char *k= ba.constData(); groupkey.set_data((void*)k); groupkey.set_size(articleIndex.length()); Db* groupsDb = ng->getGroupingDb(); ret=groupsDb->get(NULL, &groupkey, &groupdata, 0); if (ret != 0) //key not found { qDebug() << "Failed to find group with key " << articleIndex; } else { qDebug() << "Found group with key " << articleIndex; hg=new HeaderGroup(articleIndex.length(), (char*)k, (char*)groupdata.get_data()); void* ptr = groupdata.get_data(); Q_FREE(ptr); } return hg; }
/*=========================================================================== FUNCTION Q_LAST_CHECK DESCRIPTION This function returns a pointer to the link of the item at the tail of the specified queue. The item is not removed from the queue. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. RETURN VALUE The link of the last item in the queue, or NULL if there are no items on the queue. SIDE EFFECTS Pointer to link of an item on the queue is returned without de-queuing the item. ===========================================================================*/ void *q_last_check ( q_type *q_ptr /* Queue from which the item is returned */ ) { q_link_type *link_ptr = NULL; /* Link to be returned */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); /*------------------------------------------------------------------------- If there are items on the queue, return the last item -------------------------------------------------------------------------*/ Q_LOCK( q_ptr ); if( 0 < q_ptr->link_cnt ) { link_ptr = q_ptr->tail_link_ptr; } Q_FREE( q_ptr ); return (void *)link_ptr; } /* q_last_check */
/*=========================================================================== FUNCTION Q_NEXT DESCRIPTION This function returns a pointer to the item link which comes after the specified item on the specified queue. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. The specified link should have been acquired from a previous call to q_check/q_next. RETURN VALUE A pointer to the next item on the queue. If the end of the queue is reached then NULL is returned. SIDE EFFECTS Returns a pointer to an item in the queue without de-queuing it. ===========================================================================*/ void *q_next ( q_type *q_ptr, /* Queue from which item is returned */ q_link_type *link_ptr /* Link whose next link is required */ ) { q_link_type *ret_link_ptr = NULL; /* Link to be returned */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); ASSERT( NULL != link_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- If the passed link is the last link on the queue return NULL, otherwise return the link after the passed link. -------------------------------------------------------------------------*/ if( link_ptr != q_ptr->tail_link_ptr ) { ret_link_ptr = link_ptr->next_link_ptr; } Q_FREE( q_ptr ); return (void *)ret_link_ptr; } /* q_next() */
/*=========================================================================== FUNCTION Q_CHECK DESCRIPTION This function returns a pointer to the link of the item at the head of the queue. The item is not removed from the queue. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. RETURN VALUE A pointer to the first queued item's link. If the specified queue is empty, then NULL is returned. SIDE EFFECTS Pointer to link of an item in the queue is returned without de-queuing the item. ===========================================================================*/ void *q_check ( q_type *q_ptr /* Pointer to a queue */ ) { q_link_type *link_ptr = NULL; /* link pointer to be returned */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameter -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); /*------------------------------------------------------------------------- Return the first link if there is an item on the queue. -------------------------------------------------------------------------*/ Q_LOCK( q_ptr ); if( 0 < q_ptr->link_cnt ) { link_ptr = q_ptr->head_link.next_link_ptr; } Q_FREE( q_ptr ); return (void *)link_ptr; } /* q_check() */
/*=========================================================================== FUNCTION Q_PUT DESCRIPTION This function enqueues an item onto a specified queue using a specified link. The item is enqueued at the end of the queue. DEPENDENCIES The specified queue should have been previously initialized via a call to q_init. The specified link field of the item should have been prev- iously initialized via a call to q_link. RETURN VALUE None. SIDE EFFECTS The specified item is placed at the tail of the specified queue. ===========================================================================*/ void q_put ( q_type *q_ptr, /* Queue on which item is to be queued. */ q_link_type *link_ptr /* Link to use for queueing item. */ ) { /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); ASSERT( NULL != link_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- link_ptr is made to point to the queue's head_link, the queue's tail_link_ptr and the last link are made to point to link_ptr which then becomes the last link on the queue. Increment count of items on queue. -------------------------------------------------------------------------*/ link_ptr->next_link_ptr = &q_ptr->head_link; q_ptr->tail_link_ptr = q_ptr->tail_link_ptr->next_link_ptr = link_ptr; q_ptr->link_cnt++; Q_FREE( q_ptr ); return; } /* q_put() */
/*=========================================================================== FUNCTION Q_LAST_GET DESCRIPTION This function removes an item from the tail of a specified queue and returns it's link. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. RETURN VALUE The link of the last item in the queue. A NULL is returned if there is no item on the queue. SIDE EFFECTS The tail item is removed from the specified queue. ===========================================================================*/ void *q_last_get ( q_type *q_ptr /* Queue from which the item is returned */ ) { q_link_type *prev_link_ptr = NULL; /* Predecessor to the last link */ q_link_type *ret_link_ptr = NULL; /* Link to be returned */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- Check if there are any items on the queue -------------------------------------------------------------------------*/ if( &q_ptr->head_link != q_ptr->tail_link_ptr ) { /*----------------------------------------------------------------------- Get the last link and it's predecessor on the queue. Remove the last item from the queue. The ASSERT fires if the link is not on the queue and QUEUE_STRICT_CHECK is turned on. -----------------------------------------------------------------------*/ ret_link_ptr = q_ptr->tail_link_ptr; prev_link_ptr = q_prev( q_ptr, ret_link_ptr ); QUEUE_ASSERT( NULL != ret_link_ptr ); ret_link_ptr->next_link_ptr = NULL; /*----------------------------------------------------------------------- If this was the only element then set prev_link_ptr to the head_link of the queue. -----------------------------------------------------------------------*/ if( NULL == prev_link_ptr ) { prev_link_ptr = &q_ptr->head_link; } /*----------------------------------------------------------------------- Make the last link's predecessor the last link in the queue. Decrement item count. -----------------------------------------------------------------------*/ prev_link_ptr->next_link_ptr = &q_ptr->head_link; q_ptr->tail_link_ptr = prev_link_ptr; q_ptr->link_cnt--; } Q_FREE( q_ptr ); return (void *)ret_link_ptr; } /* q_last_get() */
/*=========================================================================== FUNCTION Q_INSERT DESCRIPTION This function inserts a specified item insert_link_ptr) before another specified item (next_link_ptr) on a specified queue (q_ptr). DEPENDENCIES The specified queue should have been initialized previously via a call to q_init; insert_link_ptr and next_link_ptr should have been initialized via calls to q_link. RETURN VALUE None. SIDE EFFECTS insert_link_ptr's associated item is inserted before next_link_ptr's item. ===========================================================================*/ void q_insert ( q_type *q_ptr, /* Pointer to the queue */ q_link_type *insert_link_ptr, /* Pointer to link to be inserted */ q_link_type *next_link_ptr /* This comes after insert_link_ptr */ ) { q_link_type *prev_link_ptr = NULL; /* Link before next_link_ptr */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); ASSERT( NULL != insert_link_ptr ); ASSERT( NULL != next_link_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); /*------------------------------------------------------------------------- If the next_link_ptr is the same as the head_link ASSERT. -------------------------------------------------------------------------*/ QUEUE_ASSERT( next_link_ptr != &q_ptr->head_link ); Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- Find the predecessor of next_link_ptr. If next_link_ptr is the first link on the queue, set the prev_link_ptr to the head_link. -------------------------------------------------------------------------*/ prev_link_ptr = q_prev( q_ptr, next_link_ptr ); if( NULL == prev_link_ptr && q_ptr->head_link.next_link_ptr == next_link_ptr ) { prev_link_ptr = &q_ptr->head_link; } /*------------------------------------------------------------------------- ASSERT if next_link_ptr was not found on the queue. Else insert insert_link_ptr before next_link_ptr. -------------------------------------------------------------------------*/ QUEUE_ASSERT( NULL != prev_link_ptr ); insert_link_ptr->next_link_ptr = next_link_ptr; prev_link_ptr->next_link_ptr = insert_link_ptr; q_ptr->link_cnt++; Q_FREE( q_ptr ); return; } /* q_insert() */
/*=========================================================================== FUNCTION Q_GET DESCRIPTION This function removes an item from the head of a specified queue and returns it's link. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. RETURN VALUE A pointer to the first item's link. If the specified queue is empty, then NULL is returned. SIDE EFFECTS The head item, if any, is removed from the specified queue. ===========================================================================*/ void *q_get ( q_type *q_ptr /* Queue from which the item is returned */ ) { q_link_type *link_ptr = NULL; /* The link to be returned */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- Can only get an item if the queue is non empty -------------------------------------------------------------------------*/ if( q_ptr->head_link.next_link_ptr != &q_ptr->head_link ) { /*----------------------------------------------------------------------- Get the first queued item. Make queue's head_link point to the link this item is pointing to. -----------------------------------------------------------------------*/ link_ptr = q_ptr->head_link.next_link_ptr; q_ptr->head_link.next_link_ptr = link_ptr->next_link_ptr; /*----------------------------------------------------------------------- If this is the only item then adjust queue's tail_link_ptr. -----------------------------------------------------------------------*/ if( link_ptr == q_ptr->tail_link_ptr ) { q_ptr->tail_link_ptr = link_ptr->next_link_ptr; } /*----------------------------------------------------------------------- Decrement queue count. Mark item as no longer in the queue. -----------------------------------------------------------------------*/ q_ptr->link_cnt--; link_ptr->next_link_ptr = NULL; } Q_FREE( q_ptr ); return (void *)link_ptr; } /* q_get() */
bool HeaderReadWorker::cacheFlush(uint size) { Dbt key, data; int ret; RawHeader* h = 0; QString cIndex; NewsGroup* ng = job->ng; Db* pdb = ng->getPartsDb(); // get the parts Db MultiPartHeader* mph = 0; SinglePartHeader* sph = 0; HeaderBase* hb = 0; quint16 capPart = 0; quint16 capTotal = 0; if (size == 0) size = headerCache.count(); qDebug() << "Flushing cache"; qDebug() << "cacheIndex is " << headerCache.count() << " .Flushing " << size << " records"; //Flush "size" elements, with a FIFO policy if ((int) size > headerCache.count()) { qDebug() << "wrong flush size!"; size = headerCache.count(); } uint partsAdded = size; QMutexLocker locker(headerDbLock); quint64 mPHKey = ng->getMultiPartSequence(); //qDebug() << "Multi Part Sequence in cache is " << mPHKey << " for host " << hostId; for (uint i = 0; i < size; i++) { h = headerCache.dequeue(); if (h == 0) // didn't exist { //qDebug() << "Failed to find header in cache flush !! i = " << i << ", size = " << size; continue; } pos = 0; capPart = 0; capTotal = 0; index = -1; //qDebug() << "Subject in flush is " << h->m_subj; while ((pos = rx.indexIn(h->m_subj, pos)) != -1) { index = pos; pos += rx.matchedLength(); capPart = rx.cap(1).toInt(); capTotal = rx.cap(2).toInt(); } if (index == -1) // It looks like a single part header { cIndex = h->m_subj.left(index).simplified() % '\n' % h->m_mid; // This is the common part of the subject + separator + msgId sph = (SinglePartHeader*)(cache.value(cIndex)); if (sph) // This is a duplicate (We've already seen this subj and msgId for this server ... { partsAdded--; //qDebug() << "Duplicate article in cache : " << sph->getSubj() << "for server " << hostId; } else { //Try to get the header from the db... sph = (SinglePartHeader*)dbBinHeaderGet(cIndex); if (sph) { if (!sph->isServerPresent(hostId)) // Article not currently registered to this server { sph->setServerPartNum(hostId, h->m_num); cache.insert(cIndex, sph); //qDebug() << "Server " << hostId << ", db find"; } else // This is a duplicate (We've already seen this subj and msgId for this server ... { partsAdded--; //qDebug() << "Duplicate article in db : " << sph->getSubj() << "for server " << hostId; } } else { //qDebug() << "Server " << hostId << ", sph create"; //Create new header and put it in the cache... sph = new SinglePartHeader; groupArticles++; //The new article is, of course, unread... unreadArticles++; sph->setSubj(h->m_subj.left(index).simplified()); sph->setFrom(h->m_from); sph->setStatus(MultiPartHeader::bh_new); sph->setLines(h->m_lines.toLong()); sph->setSize(h->m_bytes.toLongLong()); if (h->m_date.isNull()) qDebug() << "Date is null!!!"; else sph->setPostingDate(createDateTime(h->m_date.split(" "))); sph->setDownloadDate(QDateTime::currentDateTime().toTime_t()); sph->setMessageId(h->m_mid); sph->setServerPartNum(hostId, h->m_num); cache.insert(cIndex, sph); ng->setHeadersNeedGrouping(true); } } } else { cIndex = h->m_subj.left(index).simplified() % '\n' % h->m_from; // This is the common part of the subject + separator + sender mph = (MultiPartHeader*)(cache.value(cIndex)); if (mph == 0) { //Try to get the header from the db... //mph = dynamic_cast<MultiPartHeader*>(dbBinHeaderGet(db, cIndex)); mph = (MultiPartHeader*)(dbBinHeaderGet(cIndex)); if (mph) { //qDebug() << "Server " << hostId << ", db find"; //qDebug() << "Using mph with rec type " << mph->getHeaderType(); //qDebug() << "Using mph with key " << mph->multiPartKey; //update the header in the cache... switch (mph->addPart(pdb, capPart, h, hostId)) { case MultiPartHeader::Duplicate_Part: case MultiPartHeader::Error_Part: partsAdded--; break; case MultiPartHeader::Unread_Status: //Added a part that changed the status of the post... unreadArticles++; break; case MultiPartHeader::New_Part: case MultiPartHeader::Updated_Part: break; } cache.insert(cIndex, mph); } } else { //qDebug() << "Server " << hostId << ", cache find"; //qDebug() << "Using mph with key " << mph->multiPartKey; //update the header in the cache... switch (mph->addPart(pdb, capPart, h, hostId)) { case MultiPartHeader::Duplicate_Part: case MultiPartHeader::Error_Part: partsAdded--; break; case MultiPartHeader::Unread_Status: //Added a part that changed the status of the post... unreadArticles++; break; case MultiPartHeader::New_Part: case MultiPartHeader::Updated_Part: break; } } if (mph == 0) { //qDebug() << "Server " << hostId << ", mph create"; //Create new header and put it in the cache... mph = new MultiPartHeader; groupArticles++; //The new article is, of course, unread... unreadArticles++; mph->setSubj(h->m_subj.left(index).simplified()); mph->setFrom(h->m_from); mph->setStatus(MultiPartHeader::bh_new); mph->setKey(++mPHKey); //qDebug() << "Created mph with key " << mPHKey; if (h->m_date.isNull()) qDebug() << "Date is null!!!"; else mph->setPostingDate(createDateTime(h->m_date.split(" "))); mph->setDownloadDate(QDateTime::currentDateTime().toTime_t()); mph->setNumParts(capTotal); // Also sets missing parts to capTotal mph->addPart(pdb, capPart, h, hostId); cache.insert(cIndex, mph); ng->setHeadersNeedGrouping(true); } } Q_DELETE(h); } // Multiple threads are still controlled by mutex at this point ng->servLocalParts[hostId] += size; qDebug() << "Server " << hostId << ", total articles = " << ng->totalArticles << ", adding " << groupArticles; ng->totalArticles += groupArticles; ng->unreadArticles += unreadArticles; qDebug() << "Server " << hostId << ", total articles = " << ng->totalArticles; ng->setMultiPartSequence(mPHKey); groupArticles = 0; unreadArticles = 0; // sync the parts db pdb->sync(0); //Flush the cache that we've just built QHash<QString, HeaderBase*>::iterator it = cache.begin(); QByteArray ba; const char *cIndexCharArr = 0; while (it != cache.end()) { cIndex = it.key(); hb = (HeaderBase*)(it.value()); if (hb->getHeaderType() == 'm') { mph = (MultiPartHeader*)(it.value()); data.set_data(mph->data()); data.set_size(mph->getRecordSize()); //qDebug() << "Just saved mph with key " << mph->multiPartKey; } else { sph = (SinglePartHeader*)(it.value()); data.set_data(sph->data()); data.set_size(sph->getRecordSize()); } ba = cIndex.toLocal8Bit(); cIndexCharArr = ba.constData(); key.set_data((void*) cIndexCharArr); key.set_size(cIndex.length()); ret = db->put(NULL, &key, &data, 0); if (ret != 0) { qDebug() << "Error flushing cache: " << ret; // errorString=g_dbenv->strerror(ret); errorString = "Failure whilst writing header record to database"; return false; } void *ptr = data.get_data(); Q_FREE(ptr); if (hb->getHeaderType() == 'm') { Q_DELETE(mph); } else { Q_DELETE(sph); } ++it; } locker.unlock(); // *************** this is a massive hold *********************** cache.clear(); qDebug() << "Ok, cache flushed"; cacheFlushed = true; return true; }
/*=========================================================================== FUNCTION Q_DELETE DESCRIPTION This function removes a specified item from a specified queue. DEPENDENCIES The specified queue should have been initialized previously via a call to q_init. The specified link should have been initialized via call to q_link. RETURN VALUE None. SIDE EFFECTS Item is deleted from the queue. ===========================================================================*/ void q_delete ( q_type *q_ptr, /* Pointer to the queue */ q_link_type *delete_link_ptr /* Pointer to link to be removed from q */ ) { q_link_type *prev_link_ptr = NULL; /* Predecessor of delete_link_ptr */ /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/ /*------------------------------------------------------------------------- Verify parameters -------------------------------------------------------------------------*/ ASSERT( NULL != q_ptr ); ASSERT( NULL != delete_link_ptr ); /*------------------------------------------------------------------------- Check that the queue is initialized. Works if FEATURE_QUEUE_NO_STRICT_CHECK is turned off. -------------------------------------------------------------------------*/ QUEUE_CHECK_INIT( q_ptr ); /*------------------------------------------------------------------------- Check if the item to be deleted is head_link, if not then delete the item else assert. -------------------------------------------------------------------------*/ QUEUE_ASSERT( delete_link_ptr != &q_ptr->head_link ) Q_LOCK( q_ptr ); /*------------------------------------------------------------------------- Find the predecessor of the item to be deleted. -------------------------------------------------------------------------*/ prev_link_ptr = q_prev( q_ptr, delete_link_ptr ); /*------------------------------------------------------------------------- If this is the first item set prev_link_ptr to head_link. -------------------------------------------------------------------------*/ if( NULL == prev_link_ptr && q_ptr->head_link.next_link_ptr == delete_link_ptr ) { prev_link_ptr = &q_ptr->head_link; } /*------------------------------------------------------------------------- If we found the link on the queue, remove the item. -------------------------------------------------------------------------*/ if( NULL != prev_link_ptr ) { prev_link_ptr->next_link_ptr = delete_link_ptr->next_link_ptr; /*----------------------------------------------------------------------- If this is the last item, fix the queue's tail_link_ptr. -----------------------------------------------------------------------*/ if( delete_link_ptr == q_ptr->tail_link_ptr ) { q_ptr->tail_link_ptr = prev_link_ptr; } q_ptr->link_cnt--; delete_link_ptr->next_link_ptr = NULL; } Q_FREE( q_ptr ); return; } /* q_delete() */