void NZBFile::CalcHashes()
{
TempFileList fileList;
for (FileList::iterator it = m_pNZBInfo->GetFileList()->begin(); it != m_pNZBInfo->GetFileList()->end(); it++)
{
fileList.push_back(*it);
}
fileList.sort(CompareFileInfo);
unsigned int iFullContentHash = 0;
unsigned int iFilteredContentHash = 0;
int iUseForFilteredCount = 0;
for (TempFileList::iterator it = fileList.begin(); it != fileList.end(); it++)
{
FileInfo* pFileInfo = *it;
// check file extension
bool bSkip = !pFileInfo->GetParFile() &&
Util::MatchFileExt(pFileInfo->GetFilename(), g_pOptions->GetExtCleanupDisk(), ",;");
for (FileInfo::Articles::iterator it = pFileInfo->GetArticles()->begin(); it != pFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pArticle = *it;
int iLen = strlen(pArticle->GetMessageID());
iFullContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFullContentHash);
if (!bSkip)
{
iFilteredContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFilteredContentHash);
iUseForFilteredCount++;
}
}
}
// if filtered hash is based on less than a half of files - do not use filtered hash at all
if (iUseForFilteredCount < (int)fileList.size() / 2)
{
iFilteredContentHash = 0;
}
m_pNZBInfo->SetFullContentHash(iFullContentHash);
m_pNZBInfo->SetFilteredContentHash(iFilteredContentHash);
}
/*
* Creates new nzb-item out of existing files from other nzb-items.
* If any of file-items is being downloaded the command fail.
* For each file-item an event "eaFileDeleted" is fired.
*/
bool QueueCoordinator::SplitQueueEntries(DownloadQueue* pDownloadQueue, FileList* pFileList, const char* szName, NZBInfo** pNewNZBInfo)
{
if (pFileList->empty())
{
return false;
}
NZBInfo* pSrcNZBInfo = NULL;
for (FileList::iterator it = pFileList->begin(); it != pFileList->end(); it++)
{
FileInfo* pFileInfo = *it;
if (pFileInfo->GetActiveDownloads() > 0 || pFileInfo->GetCompletedArticles() > 0)
{
error("Could not split %s. File is already (partially) downloaded", pFileInfo->GetFilename());
return false;
}
if (pFileInfo->GetNZBInfo()->GetPostInfo())
{
error("Could not split %s. File in post-process-stage", pFileInfo->GetFilename());
return false;
}
if (!pSrcNZBInfo)
{
pSrcNZBInfo = pFileInfo->GetNZBInfo();
}
}
NZBInfo* pNZBInfo = new NZBInfo();
pDownloadQueue->GetQueue()->push_back(pNZBInfo);
pNZBInfo->SetFilename(pSrcNZBInfo->GetFilename());
pNZBInfo->SetName(szName);
pNZBInfo->SetCategory(pSrcNZBInfo->GetCategory());
pNZBInfo->SetFullContentHash(0);
pNZBInfo->SetFilteredContentHash(0);
pNZBInfo->SetPriority(pSrcNZBInfo->GetPriority());
pNZBInfo->BuildDestDirName();
pNZBInfo->SetQueuedFilename(pSrcNZBInfo->GetQueuedFilename());
pNZBInfo->GetParameters()->CopyFrom(pSrcNZBInfo->GetParameters());
pSrcNZBInfo->SetFullContentHash(0);
pSrcNZBInfo->SetFilteredContentHash(0);
for (FileList::iterator it = pFileList->begin(); it != pFileList->end(); it++)
{
FileInfo* pFileInfo = *it;
DownloadQueue::Aspect aspect = { DownloadQueue::eaFileDeleted, pDownloadQueue, pFileInfo->GetNZBInfo(), pFileInfo };
pDownloadQueue->Notify(&aspect);
pFileInfo->SetNZBInfo(pNZBInfo);
pNZBInfo->GetFileList()->push_back(pFileInfo);
pSrcNZBInfo->GetFileList()->Remove(pFileInfo);
pSrcNZBInfo->SetFileCount(pSrcNZBInfo->GetFileCount() - 1);
pSrcNZBInfo->SetSize(pSrcNZBInfo->GetSize() - pFileInfo->GetSize());
pSrcNZBInfo->SetRemainingSize(pSrcNZBInfo->GetRemainingSize() - pFileInfo->GetRemainingSize());
pSrcNZBInfo->SetCurrentSuccessSize(pSrcNZBInfo->GetCurrentSuccessSize() - pFileInfo->GetSuccessSize());
pSrcNZBInfo->SetCurrentFailedSize(pSrcNZBInfo->GetCurrentFailedSize() - pFileInfo->GetFailedSize() - pFileInfo->GetMissedSize());
pSrcNZBInfo->SetTotalArticles(pSrcNZBInfo->GetTotalArticles() - pFileInfo->GetTotalArticles());
pSrcNZBInfo->SetCurrentSuccessArticles(pSrcNZBInfo->GetCurrentSuccessArticles() - pFileInfo->GetSuccessArticles());
pSrcNZBInfo->SetCurrentFailedArticles(pSrcNZBInfo->GetCurrentFailedArticles() - pFileInfo->GetFailedArticles());
pSrcNZBInfo->GetCurrentServerStats()->ListOp(pFileInfo->GetServerStats(), ServerStatList::soSubtract);
pNZBInfo->SetFileCount(pNZBInfo->GetFileCount() + 1);
pNZBInfo->SetSize(pNZBInfo->GetSize() + pFileInfo->GetSize());
pNZBInfo->SetRemainingSize(pNZBInfo->GetRemainingSize() + pFileInfo->GetRemainingSize());
pNZBInfo->SetCurrentSuccessSize(pNZBInfo->GetCurrentSuccessSize() + pFileInfo->GetSuccessSize());
pNZBInfo->SetCurrentFailedSize(pNZBInfo->GetCurrentFailedSize() + pFileInfo->GetFailedSize() + pFileInfo->GetMissedSize());
pNZBInfo->SetTotalArticles(pNZBInfo->GetTotalArticles() + pFileInfo->GetTotalArticles());
pNZBInfo->SetCurrentSuccessArticles(pNZBInfo->GetCurrentSuccessArticles() + pFileInfo->GetSuccessArticles());
pNZBInfo->SetCurrentFailedArticles(pNZBInfo->GetCurrentFailedArticles() + pFileInfo->GetFailedArticles());
pNZBInfo->GetCurrentServerStats()->ListOp(pFileInfo->GetServerStats(), ServerStatList::soAdd);
if (pFileInfo->GetParFile())
{
pSrcNZBInfo->SetParSize(pSrcNZBInfo->GetParSize() - pFileInfo->GetSize());
pSrcNZBInfo->SetParCurrentSuccessSize(pSrcNZBInfo->GetParCurrentSuccessSize() - pFileInfo->GetSuccessSize());
pSrcNZBInfo->SetParCurrentFailedSize(pSrcNZBInfo->GetParCurrentFailedSize() - pFileInfo->GetFailedSize() - pFileInfo->GetMissedSize());
pSrcNZBInfo->SetRemainingParCount(pSrcNZBInfo->GetRemainingParCount() - 1);
pNZBInfo->SetParSize(pNZBInfo->GetParSize() + pFileInfo->GetSize());
pNZBInfo->SetParCurrentSuccessSize(pNZBInfo->GetParCurrentSuccessSize() + pFileInfo->GetSuccessSize());
pNZBInfo->SetParCurrentFailedSize(pNZBInfo->GetParCurrentFailedSize() + pFileInfo->GetFailedSize() + pFileInfo->GetMissedSize());
pNZBInfo->SetRemainingParCount(pNZBInfo->GetRemainingParCount() + 1);
}
if (pFileInfo->GetPaused())
{
pSrcNZBInfo->SetPausedFileCount(pSrcNZBInfo->GetPausedFileCount() - 1);
pSrcNZBInfo->SetPausedSize(pSrcNZBInfo->GetPausedSize() - pFileInfo->GetRemainingSize());
pNZBInfo->SetPausedFileCount(pSrcNZBInfo->GetPausedFileCount() + 1);
pNZBInfo->SetPausedSize(pNZBInfo->GetPausedSize() + pFileInfo->GetRemainingSize());
}
}
pNZBInfo->UpdateMinMaxTime();
if (pSrcNZBInfo->GetCompletedFiles()->empty())
{
pSrcNZBInfo->UpdateMinMaxTime();
}
if (pSrcNZBInfo->GetFileList()->empty())
{
pDownloadQueue->GetQueue()->Remove(pSrcNZBInfo);
g_pDiskState->DiscardFiles(pSrcNZBInfo);
delete pSrcNZBInfo;
}
*pNewNZBInfo = pNZBInfo;
return true;
}
void QueueCoordinator::ArticleCompleted(ArticleDownloader* pArticleDownloader)
{
debug("Article downloaded");
FileInfo* pFileInfo = pArticleDownloader->GetFileInfo();
NZBInfo* pNZBInfo = pFileInfo->GetNZBInfo();
ArticleInfo* pArticleInfo = pArticleDownloader->GetArticleInfo();
bool bRetry = false;
bool fileCompleted = false;
DownloadQueue* pDownloadQueue = DownloadQueue::Lock();
if (pArticleDownloader->GetStatus() == ArticleDownloader::adFinished)
{
pArticleInfo->SetStatus(ArticleInfo::aiFinished);
pFileInfo->SetSuccessSize(pFileInfo->GetSuccessSize() + pArticleInfo->GetSize());
pNZBInfo->SetCurrentSuccessSize(pNZBInfo->GetCurrentSuccessSize() + pArticleInfo->GetSize());
pNZBInfo->SetParCurrentSuccessSize(pNZBInfo->GetParCurrentSuccessSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0));
pFileInfo->SetSuccessArticles(pFileInfo->GetSuccessArticles() + 1);
pNZBInfo->SetCurrentSuccessArticles(pNZBInfo->GetCurrentSuccessArticles() + 1);
}
else if (pArticleDownloader->GetStatus() == ArticleDownloader::adFailed)
{
pArticleInfo->SetStatus(ArticleInfo::aiFailed);
pFileInfo->SetFailedSize(pFileInfo->GetFailedSize() + pArticleInfo->GetSize());
pNZBInfo->SetCurrentFailedSize(pNZBInfo->GetCurrentFailedSize() + pArticleInfo->GetSize());
pNZBInfo->SetParCurrentFailedSize(pNZBInfo->GetParCurrentFailedSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0));
pFileInfo->SetFailedArticles(pFileInfo->GetFailedArticles() + 1);
pNZBInfo->SetCurrentFailedArticles(pNZBInfo->GetCurrentFailedArticles() + 1);
}
else if (pArticleDownloader->GetStatus() == ArticleDownloader::adRetry)
{
pArticleInfo->SetStatus(ArticleInfo::aiUndefined);
bRetry = true;
}
if (!bRetry)
{
pFileInfo->SetRemainingSize(pFileInfo->GetRemainingSize() - pArticleInfo->GetSize());
pNZBInfo->SetRemainingSize(pNZBInfo->GetRemainingSize() - pArticleInfo->GetSize());
if (pFileInfo->GetPaused())
{
pNZBInfo->SetPausedSize(pNZBInfo->GetPausedSize() - pArticleInfo->GetSize());
}
pFileInfo->SetCompletedArticles(pFileInfo->GetCompletedArticles() + 1);
fileCompleted = (int)pFileInfo->GetArticles()->size() == pFileInfo->GetCompletedArticles();
pFileInfo->GetServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd);
pNZBInfo->GetCurrentServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd);
pFileInfo->SetPartialChanged(true);
}
if (!pFileInfo->GetFilenameConfirmed() &&
pArticleDownloader->GetStatus() == ArticleDownloader::adFinished &&
pArticleDownloader->GetArticleFilename())
{
pFileInfo->SetFilename(pArticleDownloader->GetArticleFilename());
pFileInfo->SetFilenameConfirmed(true);
if (g_pOptions->GetDupeCheck() &&
pNZBInfo->GetDupeMode() != dmForce &&
!pNZBInfo->GetManyDupeFiles() &&
Util::FileExists(pNZBInfo->GetDestDir(), pFileInfo->GetFilename()))
{
warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", pFileInfo->GetFilename());
fileCompleted = false;
pFileInfo->SetAutoDeleted(true);
DeleteQueueEntry(pDownloadQueue, pFileInfo);
}
}
pNZBInfo->SetDownloadedSize(pNZBInfo->GetDownloadedSize() + pArticleDownloader->GetDownloadedSize());
bool deleteFileObj = false;
if (fileCompleted && !pFileInfo->GetDeleted())
{
// all jobs done
DownloadQueue::Unlock();
pArticleDownloader->CompleteFileParts();
pDownloadQueue = DownloadQueue::Lock();
deleteFileObj = true;
}
CheckHealth(pDownloadQueue, pFileInfo);
bool hasOtherDownloaders = false;
for (ActiveDownloads::iterator it = m_ActiveDownloads.begin(); it != m_ActiveDownloads.end(); it++)
{
ArticleDownloader* pDownloader = *it;
if (pDownloader != pArticleDownloader && pDownloader->GetFileInfo() == pFileInfo)
{
hasOtherDownloaders = true;
break;
}
}
deleteFileObj |= pFileInfo->GetDeleted() && !hasOtherDownloaders;
// remove downloader from downloader list
m_ActiveDownloads.erase(std::find(m_ActiveDownloads.begin(), m_ActiveDownloads.end(), pArticleDownloader));
pFileInfo->SetActiveDownloads(pFileInfo->GetActiveDownloads() - 1);
pNZBInfo->SetActiveDownloads(pNZBInfo->GetActiveDownloads() - 1);
if (deleteFileObj)
{
DeleteFileInfo(pDownloadQueue, pFileInfo, fileCompleted);
pDownloadQueue->Save();
}
DownloadQueue::Unlock();
}
/**
* Unpause par2-files
* returns true, if the files with required number of blocks were unpaused,
* or false if there are no more files in queue for this collection or not enough blocks.
* special case: returns true if there are any unpaused par2-files in the queue regardless
* of the amount of blocks; this is to keep par-checker wait for download completion.
*/
bool ParCoordinator::RequestMorePars(NZBInfo* pNZBInfo, const char* szParFilename, int iBlockNeeded, int* pBlockFound)
{
DownloadQueue* pDownloadQueue = DownloadQueue::Lock();
Blocks blocks;
blocks.clear();
int iBlockFound = 0;
int iCurBlockFound = 0;
FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, true, true, &iCurBlockFound);
iBlockFound += iCurBlockFound;
if (iBlockFound < iBlockNeeded)
{
FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, true, false, &iCurBlockFound);
iBlockFound += iCurBlockFound;
}
if (iBlockFound < iBlockNeeded)
{
FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, false, false, &iCurBlockFound);
iBlockFound += iCurBlockFound;
}
if (iBlockFound >= iBlockNeeded)
{
// 1. first unpause all files with par-blocks less or equal iBlockNeeded
// starting from the file with max block count.
// if par-collection was built exponentially and all par-files present,
// this step selects par-files with exact number of blocks we need.
while (iBlockNeeded > 0)
{
BlockInfo* pBestBlockInfo = NULL;
for (Blocks::iterator it = blocks.begin(); it != blocks.end(); it++)
{
BlockInfo* pBlockInfo = *it;
if (pBlockInfo->m_iBlockCount <= iBlockNeeded &&
(!pBestBlockInfo || pBestBlockInfo->m_iBlockCount < pBlockInfo->m_iBlockCount))
{
pBestBlockInfo = pBlockInfo;
}
}
if (pBestBlockInfo)
{
if (pBestBlockInfo->m_pFileInfo->GetPaused())
{
m_ParChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", pNZBInfo->GetName(), (int)PATH_SEPARATOR, pBestBlockInfo->m_pFileInfo->GetFilename());
pBestBlockInfo->m_pFileInfo->SetPaused(false);
pBestBlockInfo->m_pFileInfo->SetExtraPriority(true);
}
iBlockNeeded -= pBestBlockInfo->m_iBlockCount;
blocks.remove(pBestBlockInfo);
delete pBestBlockInfo;
}
else
{
break;
}
}
// 2. then unpause other files
// this step only needed if the par-collection was built not exponentially
// or not all par-files present (or some of them were corrupted)
// this step is not optimal, but we hope, that the first step will work good
// in most cases and we will not need the second step often
while (iBlockNeeded > 0)
{
BlockInfo* pBlockInfo = blocks.front();
if (pBlockInfo->m_pFileInfo->GetPaused())
{
m_ParChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", pNZBInfo->GetName(), (int)PATH_SEPARATOR, pBlockInfo->m_pFileInfo->GetFilename());
pBlockInfo->m_pFileInfo->SetPaused(false);
pBlockInfo->m_pFileInfo->SetExtraPriority(true);
}
iBlockNeeded -= pBlockInfo->m_iBlockCount;
}
}
bool bHasUnpausedParFiles = false;
for (FileList::iterator it = pNZBInfo->GetFileList()->begin(); it != pNZBInfo->GetFileList()->end(); it++)
{
FileInfo* pFileInfo = *it;
if (pFileInfo->GetParFile() && !pFileInfo->GetPaused())
{
bHasUnpausedParFiles = true;
break;
}
}
DownloadQueue::Unlock();
if (pBlockFound)
{
*pBlockFound = iBlockFound;
}
for (Blocks::iterator it = blocks.begin(); it != blocks.end(); it++)
{
delete *it;
}
blocks.clear();
bool bOK = iBlockNeeded <= 0 || bHasUnpausedParFiles;
return bOK;
}