bool HeaderReadWorker::cacheFlush(uint size)
{
Dbt key, data;
int ret;
RawHeader* h = 0;
QString cIndex;
NewsGroup* ng = job->ng;
Db* pdb = ng->getPartsDb(); // get the parts Db
MultiPartHeader* mph = 0;
SinglePartHeader* sph = 0;
HeaderBase* hb = 0;
quint16 capPart = 0;
quint16 capTotal = 0;
if (size == 0)
size = headerCache.count();
qDebug() << "Flushing cache";
qDebug() << "cacheIndex is " << headerCache.count() << " .Flushing " << size
<< " records";
//Flush "size" elements, with a FIFO policy
if ((int) size > headerCache.count())
{
qDebug() << "wrong flush size!";
size = headerCache.count();
}
uint partsAdded = size;
QMutexLocker locker(headerDbLock);
quint64 mPHKey = ng->getMultiPartSequence();
//qDebug() << "Multi Part Sequence in cache is " << mPHKey << " for host " << hostId;
for (uint i = 0; i < size; i++)
{
h = headerCache.dequeue();
if (h == 0) // didn't exist
{
//qDebug() << "Failed to find header in cache flush !! i = " << i << ", size = " << size;
continue;
}
pos = 0;
capPart = 0;
capTotal = 0;
index = -1;
//qDebug() << "Subject in flush is " << h->m_subj;
while ((pos = rx.indexIn(h->m_subj, pos)) != -1)
{
index = pos;
pos += rx.matchedLength();
capPart = rx.cap(1).toInt();
capTotal = rx.cap(2).toInt();
}
if (index == -1) // It looks like a single part header
{
cIndex = h->m_subj.left(index).simplified() % '\n' % h->m_mid; // This is the common part of the subject + separator + msgId
sph = (SinglePartHeader*)(cache.value(cIndex));
if (sph) // This is a duplicate (We've already seen this subj and msgId for this server ...
{
partsAdded--;
//qDebug() << "Duplicate article in cache : " << sph->getSubj() << "for server " << hostId;
}
else
{
//Try to get the header from the db...
sph = (SinglePartHeader*)dbBinHeaderGet(cIndex);
if (sph)
{
if (!sph->isServerPresent(hostId)) // Article not currently registered to this server
{
sph->setServerPartNum(hostId, h->m_num);
cache.insert(cIndex, sph);
//qDebug() << "Server " << hostId << ", db find";
}
else // This is a duplicate (We've already seen this subj and msgId for this server ...
{
partsAdded--;
//qDebug() << "Duplicate article in db : " << sph->getSubj() << "for server " << hostId;
}
}
else
{
//qDebug() << "Server " << hostId << ", sph create";
//Create new header and put it in the cache...
sph = new SinglePartHeader;
groupArticles++;
//The new article is, of course, unread...
unreadArticles++;
sph->setSubj(h->m_subj.left(index).simplified());
sph->setFrom(h->m_from);
sph->setStatus(MultiPartHeader::bh_new);
sph->setLines(h->m_lines.toLong());
sph->setSize(h->m_bytes.toLongLong());
if (h->m_date.isNull())
qDebug() << "Date is null!!!";
else
sph->setPostingDate(createDateTime(h->m_date.split(" ")));
sph->setDownloadDate(QDateTime::currentDateTime().toTime_t());
sph->setMessageId(h->m_mid);
sph->setServerPartNum(hostId, h->m_num);
cache.insert(cIndex, sph);
ng->setHeadersNeedGrouping(true);
}
}
}
else
{
cIndex = h->m_subj.left(index).simplified() % '\n' % h->m_from; // This is the common part of the subject + separator + sender
mph = (MultiPartHeader*)(cache.value(cIndex));
if (mph == 0)
{
//Try to get the header from the db...
//mph = dynamic_cast<MultiPartHeader*>(dbBinHeaderGet(db, cIndex));
mph = (MultiPartHeader*)(dbBinHeaderGet(cIndex));
if (mph)
{
//qDebug() << "Server " << hostId << ", db find";
//qDebug() << "Using mph with rec type " << mph->getHeaderType();
//qDebug() << "Using mph with key " << mph->multiPartKey;
//update the header in the cache...
switch (mph->addPart(pdb, capPart, h, hostId))
{
case MultiPartHeader::Duplicate_Part:
case MultiPartHeader::Error_Part:
partsAdded--;
break;
case MultiPartHeader::Unread_Status:
//Added a part that changed the status of the post...
unreadArticles++;
break;
case MultiPartHeader::New_Part:
case MultiPartHeader::Updated_Part:
break;
}
cache.insert(cIndex, mph);
}
}
else
{
//qDebug() << "Server " << hostId << ", cache find";
//qDebug() << "Using mph with key " << mph->multiPartKey;
//update the header in the cache...
switch (mph->addPart(pdb, capPart, h, hostId))
{
case MultiPartHeader::Duplicate_Part:
case MultiPartHeader::Error_Part:
partsAdded--;
break;
case MultiPartHeader::Unread_Status:
//Added a part that changed the status of the post...
unreadArticles++;
break;
case MultiPartHeader::New_Part:
case MultiPartHeader::Updated_Part:
break;
}
}
if (mph == 0)
{
//qDebug() << "Server " << hostId << ", mph create";
//Create new header and put it in the cache...
mph = new MultiPartHeader;
groupArticles++;
//The new article is, of course, unread...
unreadArticles++;
mph->setSubj(h->m_subj.left(index).simplified());
mph->setFrom(h->m_from);
mph->setStatus(MultiPartHeader::bh_new);
mph->setKey(++mPHKey);
//qDebug() << "Created mph with key " << mPHKey;
if (h->m_date.isNull())
qDebug() << "Date is null!!!";
else
mph->setPostingDate(createDateTime(h->m_date.split(" ")));
mph->setDownloadDate(QDateTime::currentDateTime().toTime_t());
mph->setNumParts(capTotal); // Also sets missing parts to capTotal
mph->addPart(pdb, capPart, h, hostId);
cache.insert(cIndex, mph);
ng->setHeadersNeedGrouping(true);
}
}
Q_DELETE(h);
}
// Multiple threads are still controlled by mutex at this point
ng->servLocalParts[hostId] += size;
qDebug() << "Server " << hostId << ", total articles = " << ng->totalArticles << ", adding " << groupArticles;
ng->totalArticles += groupArticles;
ng->unreadArticles += unreadArticles;
qDebug() << "Server " << hostId << ", total articles = " << ng->totalArticles;
ng->setMultiPartSequence(mPHKey);
groupArticles = 0;
unreadArticles = 0;
// sync the parts db
pdb->sync(0);
//Flush the cache that we've just built
QHash<QString, HeaderBase*>::iterator it = cache.begin();
QByteArray ba;
const char *cIndexCharArr = 0;
while (it != cache.end())
{
cIndex = it.key();
hb = (HeaderBase*)(it.value());
if (hb->getHeaderType() == 'm')
{
mph = (MultiPartHeader*)(it.value());
data.set_data(mph->data());
data.set_size(mph->getRecordSize());
//qDebug() << "Just saved mph with key " << mph->multiPartKey;
}
else
{
sph = (SinglePartHeader*)(it.value());
data.set_data(sph->data());
data.set_size(sph->getRecordSize());
}
ba = cIndex.toLocal8Bit();
cIndexCharArr = ba.constData();
key.set_data((void*) cIndexCharArr);
key.set_size(cIndex.length());
ret = db->put(NULL, &key, &data, 0);
if (ret != 0)
{
qDebug() << "Error flushing cache: " << ret;
// errorString=g_dbenv->strerror(ret);
errorString = "Failure whilst writing header record to database";
return false;
}
void *ptr = data.get_data();
Q_FREE(ptr);
if (hb->getHeaderType() == 'm')
{
Q_DELETE(mph);
}
else
{
Q_DELETE(sph);
}
++it;
}
locker.unlock(); // *************** this is a massive hold ***********************
cache.clear();
qDebug() << "Ok, cache flushed";
cacheFlushed = true;
return true;
}
