void NZBFile::CalcHashes() { TempFileList fileList; for (FileList::iterator it = m_pNZBInfo->GetFileList()->begin(); it != m_pNZBInfo->GetFileList()->end(); it++) { fileList.push_back(*it); } fileList.sort(CompareFileInfo); unsigned int iFullContentHash = 0; unsigned int iFilteredContentHash = 0; int iUseForFilteredCount = 0; for (TempFileList::iterator it = fileList.begin(); it != fileList.end(); it++) { FileInfo* pFileInfo = *it; // check file extension bool bSkip = !pFileInfo->GetParFile() && Util::MatchFileExt(pFileInfo->GetFilename(), g_pOptions->GetExtCleanupDisk(), ",;"); for (FileInfo::Articles::iterator it = pFileInfo->GetArticles()->begin(); it != pFileInfo->GetArticles()->end(); it++) { ArticleInfo* pArticle = *it; int iLen = strlen(pArticle->GetMessageID()); iFullContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFullContentHash); if (!bSkip) { iFilteredContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFilteredContentHash); iUseForFilteredCount++; } } } // if filtered hash is based on less than a half of files - do not use filtered hash at all if (iUseForFilteredCount < (int)fileList.size() / 2) { iFilteredContentHash = 0; } m_pNZBInfo->SetFullContentHash(iFullContentHash); m_pNZBInfo->SetFilteredContentHash(iFilteredContentHash); }
/* * Creates new nzb-item out of existing files from other nzb-items. * If any of file-items is being downloaded the command fail. * For each file-item an event "eaFileDeleted" is fired. */ bool QueueCoordinator::SplitQueueEntries(DownloadQueue* pDownloadQueue, FileList* pFileList, const char* szName, NZBInfo** pNewNZBInfo) { if (pFileList->empty()) { return false; } NZBInfo* pSrcNZBInfo = NULL; for (FileList::iterator it = pFileList->begin(); it != pFileList->end(); it++) { FileInfo* pFileInfo = *it; if (pFileInfo->GetActiveDownloads() > 0 || pFileInfo->GetCompletedArticles() > 0) { error("Could not split %s. File is already (partially) downloaded", pFileInfo->GetFilename()); return false; } if (pFileInfo->GetNZBInfo()->GetPostInfo()) { error("Could not split %s. File in post-process-stage", pFileInfo->GetFilename()); return false; } if (!pSrcNZBInfo) { pSrcNZBInfo = pFileInfo->GetNZBInfo(); } } NZBInfo* pNZBInfo = new NZBInfo(); pDownloadQueue->GetQueue()->push_back(pNZBInfo); pNZBInfo->SetFilename(pSrcNZBInfo->GetFilename()); pNZBInfo->SetName(szName); pNZBInfo->SetCategory(pSrcNZBInfo->GetCategory()); pNZBInfo->SetFullContentHash(0); pNZBInfo->SetFilteredContentHash(0); pNZBInfo->SetPriority(pSrcNZBInfo->GetPriority()); pNZBInfo->BuildDestDirName(); pNZBInfo->SetQueuedFilename(pSrcNZBInfo->GetQueuedFilename()); pNZBInfo->GetParameters()->CopyFrom(pSrcNZBInfo->GetParameters()); pSrcNZBInfo->SetFullContentHash(0); pSrcNZBInfo->SetFilteredContentHash(0); for (FileList::iterator it = pFileList->begin(); it != pFileList->end(); it++) { FileInfo* pFileInfo = *it; DownloadQueue::Aspect aspect = { DownloadQueue::eaFileDeleted, pDownloadQueue, pFileInfo->GetNZBInfo(), pFileInfo }; pDownloadQueue->Notify(&aspect); pFileInfo->SetNZBInfo(pNZBInfo); pNZBInfo->GetFileList()->push_back(pFileInfo); pSrcNZBInfo->GetFileList()->Remove(pFileInfo); pSrcNZBInfo->SetFileCount(pSrcNZBInfo->GetFileCount() - 1); pSrcNZBInfo->SetSize(pSrcNZBInfo->GetSize() - pFileInfo->GetSize()); pSrcNZBInfo->SetRemainingSize(pSrcNZBInfo->GetRemainingSize() - pFileInfo->GetRemainingSize()); pSrcNZBInfo->SetCurrentSuccessSize(pSrcNZBInfo->GetCurrentSuccessSize() - pFileInfo->GetSuccessSize()); pSrcNZBInfo->SetCurrentFailedSize(pSrcNZBInfo->GetCurrentFailedSize() - pFileInfo->GetFailedSize() - pFileInfo->GetMissedSize()); pSrcNZBInfo->SetTotalArticles(pSrcNZBInfo->GetTotalArticles() - pFileInfo->GetTotalArticles()); pSrcNZBInfo->SetCurrentSuccessArticles(pSrcNZBInfo->GetCurrentSuccessArticles() - pFileInfo->GetSuccessArticles()); pSrcNZBInfo->SetCurrentFailedArticles(pSrcNZBInfo->GetCurrentFailedArticles() - pFileInfo->GetFailedArticles()); pSrcNZBInfo->GetCurrentServerStats()->ListOp(pFileInfo->GetServerStats(), ServerStatList::soSubtract); pNZBInfo->SetFileCount(pNZBInfo->GetFileCount() + 1); pNZBInfo->SetSize(pNZBInfo->GetSize() + pFileInfo->GetSize()); pNZBInfo->SetRemainingSize(pNZBInfo->GetRemainingSize() + pFileInfo->GetRemainingSize()); pNZBInfo->SetCurrentSuccessSize(pNZBInfo->GetCurrentSuccessSize() + pFileInfo->GetSuccessSize()); pNZBInfo->SetCurrentFailedSize(pNZBInfo->GetCurrentFailedSize() + pFileInfo->GetFailedSize() + pFileInfo->GetMissedSize()); pNZBInfo->SetTotalArticles(pNZBInfo->GetTotalArticles() + pFileInfo->GetTotalArticles()); pNZBInfo->SetCurrentSuccessArticles(pNZBInfo->GetCurrentSuccessArticles() + pFileInfo->GetSuccessArticles()); pNZBInfo->SetCurrentFailedArticles(pNZBInfo->GetCurrentFailedArticles() + pFileInfo->GetFailedArticles()); pNZBInfo->GetCurrentServerStats()->ListOp(pFileInfo->GetServerStats(), ServerStatList::soAdd); if (pFileInfo->GetParFile()) { pSrcNZBInfo->SetParSize(pSrcNZBInfo->GetParSize() - pFileInfo->GetSize()); pSrcNZBInfo->SetParCurrentSuccessSize(pSrcNZBInfo->GetParCurrentSuccessSize() - pFileInfo->GetSuccessSize()); pSrcNZBInfo->SetParCurrentFailedSize(pSrcNZBInfo->GetParCurrentFailedSize() - pFileInfo->GetFailedSize() - pFileInfo->GetMissedSize()); pSrcNZBInfo->SetRemainingParCount(pSrcNZBInfo->GetRemainingParCount() - 1); pNZBInfo->SetParSize(pNZBInfo->GetParSize() + pFileInfo->GetSize()); pNZBInfo->SetParCurrentSuccessSize(pNZBInfo->GetParCurrentSuccessSize() + pFileInfo->GetSuccessSize()); pNZBInfo->SetParCurrentFailedSize(pNZBInfo->GetParCurrentFailedSize() + pFileInfo->GetFailedSize() + pFileInfo->GetMissedSize()); pNZBInfo->SetRemainingParCount(pNZBInfo->GetRemainingParCount() + 1); } if (pFileInfo->GetPaused()) { pSrcNZBInfo->SetPausedFileCount(pSrcNZBInfo->GetPausedFileCount() - 1); pSrcNZBInfo->SetPausedSize(pSrcNZBInfo->GetPausedSize() - pFileInfo->GetRemainingSize()); pNZBInfo->SetPausedFileCount(pSrcNZBInfo->GetPausedFileCount() + 1); pNZBInfo->SetPausedSize(pNZBInfo->GetPausedSize() + pFileInfo->GetRemainingSize()); } } pNZBInfo->UpdateMinMaxTime(); if (pSrcNZBInfo->GetCompletedFiles()->empty()) { pSrcNZBInfo->UpdateMinMaxTime(); } if (pSrcNZBInfo->GetFileList()->empty()) { pDownloadQueue->GetQueue()->Remove(pSrcNZBInfo); g_pDiskState->DiscardFiles(pSrcNZBInfo); delete pSrcNZBInfo; } *pNewNZBInfo = pNZBInfo; return true; }
void QueueCoordinator::ArticleCompleted(ArticleDownloader* pArticleDownloader) { debug("Article downloaded"); FileInfo* pFileInfo = pArticleDownloader->GetFileInfo(); NZBInfo* pNZBInfo = pFileInfo->GetNZBInfo(); ArticleInfo* pArticleInfo = pArticleDownloader->GetArticleInfo(); bool bRetry = false; bool fileCompleted = false; DownloadQueue* pDownloadQueue = DownloadQueue::Lock(); if (pArticleDownloader->GetStatus() == ArticleDownloader::adFinished) { pArticleInfo->SetStatus(ArticleInfo::aiFinished); pFileInfo->SetSuccessSize(pFileInfo->GetSuccessSize() + pArticleInfo->GetSize()); pNZBInfo->SetCurrentSuccessSize(pNZBInfo->GetCurrentSuccessSize() + pArticleInfo->GetSize()); pNZBInfo->SetParCurrentSuccessSize(pNZBInfo->GetParCurrentSuccessSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0)); pFileInfo->SetSuccessArticles(pFileInfo->GetSuccessArticles() + 1); pNZBInfo->SetCurrentSuccessArticles(pNZBInfo->GetCurrentSuccessArticles() + 1); } else if (pArticleDownloader->GetStatus() == ArticleDownloader::adFailed) { pArticleInfo->SetStatus(ArticleInfo::aiFailed); pFileInfo->SetFailedSize(pFileInfo->GetFailedSize() + pArticleInfo->GetSize()); pNZBInfo->SetCurrentFailedSize(pNZBInfo->GetCurrentFailedSize() + pArticleInfo->GetSize()); pNZBInfo->SetParCurrentFailedSize(pNZBInfo->GetParCurrentFailedSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0)); pFileInfo->SetFailedArticles(pFileInfo->GetFailedArticles() + 1); pNZBInfo->SetCurrentFailedArticles(pNZBInfo->GetCurrentFailedArticles() + 1); } else if (pArticleDownloader->GetStatus() == ArticleDownloader::adRetry) { pArticleInfo->SetStatus(ArticleInfo::aiUndefined); bRetry = true; } if (!bRetry) { pFileInfo->SetRemainingSize(pFileInfo->GetRemainingSize() - pArticleInfo->GetSize()); pNZBInfo->SetRemainingSize(pNZBInfo->GetRemainingSize() - pArticleInfo->GetSize()); if (pFileInfo->GetPaused()) { pNZBInfo->SetPausedSize(pNZBInfo->GetPausedSize() - pArticleInfo->GetSize()); } pFileInfo->SetCompletedArticles(pFileInfo->GetCompletedArticles() + 1); fileCompleted = (int)pFileInfo->GetArticles()->size() == pFileInfo->GetCompletedArticles(); pFileInfo->GetServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd); pNZBInfo->GetCurrentServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd); pFileInfo->SetPartialChanged(true); } if (!pFileInfo->GetFilenameConfirmed() && pArticleDownloader->GetStatus() == ArticleDownloader::adFinished && pArticleDownloader->GetArticleFilename()) { pFileInfo->SetFilename(pArticleDownloader->GetArticleFilename()); pFileInfo->SetFilenameConfirmed(true); if (g_pOptions->GetDupeCheck() && pNZBInfo->GetDupeMode() != dmForce && !pNZBInfo->GetManyDupeFiles() && Util::FileExists(pNZBInfo->GetDestDir(), pFileInfo->GetFilename())) { warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", pFileInfo->GetFilename()); fileCompleted = false; pFileInfo->SetAutoDeleted(true); DeleteQueueEntry(pDownloadQueue, pFileInfo); } } pNZBInfo->SetDownloadedSize(pNZBInfo->GetDownloadedSize() + pArticleDownloader->GetDownloadedSize()); bool deleteFileObj = false; if (fileCompleted && !pFileInfo->GetDeleted()) { // all jobs done DownloadQueue::Unlock(); pArticleDownloader->CompleteFileParts(); pDownloadQueue = DownloadQueue::Lock(); deleteFileObj = true; } CheckHealth(pDownloadQueue, pFileInfo); bool hasOtherDownloaders = false; for (ActiveDownloads::iterator it = m_ActiveDownloads.begin(); it != m_ActiveDownloads.end(); it++) { ArticleDownloader* pDownloader = *it; if (pDownloader != pArticleDownloader && pDownloader->GetFileInfo() == pFileInfo) { hasOtherDownloaders = true; break; } } deleteFileObj |= pFileInfo->GetDeleted() && !hasOtherDownloaders; // remove downloader from downloader list m_ActiveDownloads.erase(std::find(m_ActiveDownloads.begin(), m_ActiveDownloads.end(), pArticleDownloader)); pFileInfo->SetActiveDownloads(pFileInfo->GetActiveDownloads() - 1); pNZBInfo->SetActiveDownloads(pNZBInfo->GetActiveDownloads() - 1); if (deleteFileObj) { DeleteFileInfo(pDownloadQueue, pFileInfo, fileCompleted); pDownloadQueue->Save(); } DownloadQueue::Unlock(); }
/** * Unpause par2-files * returns true, if the files with required number of blocks were unpaused, * or false if there are no more files in queue for this collection or not enough blocks. * special case: returns true if there are any unpaused par2-files in the queue regardless * of the amount of blocks; this is to keep par-checker wait for download completion. */ bool ParCoordinator::RequestMorePars(NZBInfo* pNZBInfo, const char* szParFilename, int iBlockNeeded, int* pBlockFound) { DownloadQueue* pDownloadQueue = DownloadQueue::Lock(); Blocks blocks; blocks.clear(); int iBlockFound = 0; int iCurBlockFound = 0; FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, true, true, &iCurBlockFound); iBlockFound += iCurBlockFound; if (iBlockFound < iBlockNeeded) { FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, true, false, &iCurBlockFound); iBlockFound += iCurBlockFound; } if (iBlockFound < iBlockNeeded) { FindPars(pDownloadQueue, pNZBInfo, szParFilename, &blocks, false, false, &iCurBlockFound); iBlockFound += iCurBlockFound; } if (iBlockFound >= iBlockNeeded) { // 1. first unpause all files with par-blocks less or equal iBlockNeeded // starting from the file with max block count. // if par-collection was built exponentially and all par-files present, // this step selects par-files with exact number of blocks we need. while (iBlockNeeded > 0) { BlockInfo* pBestBlockInfo = NULL; for (Blocks::iterator it = blocks.begin(); it != blocks.end(); it++) { BlockInfo* pBlockInfo = *it; if (pBlockInfo->m_iBlockCount <= iBlockNeeded && (!pBestBlockInfo || pBestBlockInfo->m_iBlockCount < pBlockInfo->m_iBlockCount)) { pBestBlockInfo = pBlockInfo; } } if (pBestBlockInfo) { if (pBestBlockInfo->m_pFileInfo->GetPaused()) { m_ParChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", pNZBInfo->GetName(), (int)PATH_SEPARATOR, pBestBlockInfo->m_pFileInfo->GetFilename()); pBestBlockInfo->m_pFileInfo->SetPaused(false); pBestBlockInfo->m_pFileInfo->SetExtraPriority(true); } iBlockNeeded -= pBestBlockInfo->m_iBlockCount; blocks.remove(pBestBlockInfo); delete pBestBlockInfo; } else { break; } } // 2. then unpause other files // this step only needed if the par-collection was built not exponentially // or not all par-files present (or some of them were corrupted) // this step is not optimal, but we hope, that the first step will work good // in most cases and we will not need the second step often while (iBlockNeeded > 0) { BlockInfo* pBlockInfo = blocks.front(); if (pBlockInfo->m_pFileInfo->GetPaused()) { m_ParChecker.PrintMessage(Message::mkInfo, "Unpausing %s%c%s for par-recovery", pNZBInfo->GetName(), (int)PATH_SEPARATOR, pBlockInfo->m_pFileInfo->GetFilename()); pBlockInfo->m_pFileInfo->SetPaused(false); pBlockInfo->m_pFileInfo->SetExtraPriority(true); } iBlockNeeded -= pBlockInfo->m_iBlockCount; } } bool bHasUnpausedParFiles = false; for (FileList::iterator it = pNZBInfo->GetFileList()->begin(); it != pNZBInfo->GetFileList()->end(); it++) { FileInfo* pFileInfo = *it; if (pFileInfo->GetParFile() && !pFileInfo->GetPaused()) { bHasUnpausedParFiles = true; break; } } DownloadQueue::Unlock(); if (pBlockFound) { *pBlockFound = iBlockFound; } for (Blocks::iterator it = blocks.begin(); it != blocks.end(); it++) { delete *it; } blocks.clear(); bool bOK = iBlockNeeded <= 0 || bHasUnpausedParFiles; return bOK; }