void NZBFile::AddFileInfo(FileInfo* pFileInfo)
{
// calculate file size and delete empty articles
long long lSize = 0;
long long lMissedSize = 0;
long long lOneSize = 0;
int iUncountedArticles = 0;
int iMissedArticles = 0;
FileInfo::Articles* pArticles = pFileInfo->GetArticles();
int iTotalArticles = (int)pArticles->size();
int i = 0;
for (FileInfo::Articles::iterator it = pArticles->begin(); it != pArticles->end(); )
{
ArticleInfo* pArticle = *it;
if (!pArticle)
{
pArticles->erase(it);
it = pArticles->begin() + i;
iMissedArticles++;
if (lOneSize > 0)
{
lMissedSize += lOneSize;
}
else
{
iUncountedArticles++;
}
}
else
{
lSize += pArticle->GetSize();
if (lOneSize == 0)
{
lOneSize = pArticle->GetSize();
}
it++;
i++;
}
}
if (pArticles->empty())
{
delete pFileInfo;
return;
}
lMissedSize += iUncountedArticles * lOneSize;
lSize += lMissedSize;
m_pNZBInfo->GetFileList()->push_back(pFileInfo);
pFileInfo->SetNZBInfo(m_pNZBInfo);
pFileInfo->SetSize(lSize);
pFileInfo->SetRemainingSize(lSize - lMissedSize);
pFileInfo->SetMissedSize(lMissedSize);
pFileInfo->SetTotalArticles(iTotalArticles);
pFileInfo->SetMissedArticles(iMissedArticles);
}
void QueueCoordinator::ResetHangingDownloads()
{
if (g_pOptions->GetTerminateTimeout() == 0 && g_pOptions->GetArticleTimeout() == 0)
{
return;
}
DownloadQueue::Lock();
time_t tm = ::time(NULL);
for (ActiveDownloads::iterator it = m_ActiveDownloads.begin(); it != m_ActiveDownloads.end();)
{
ArticleDownloader* pArticleDownloader = *it;
if (tm - pArticleDownloader->GetLastUpdateTime() > g_pOptions->GetArticleTimeout() + 1 &&
pArticleDownloader->GetStatus() == ArticleDownloader::adRunning)
{
error("Cancelling hanging download %s @ %s", pArticleDownloader->GetInfoName(),
pArticleDownloader->GetConnectionName());
pArticleDownloader->Stop();
}
if (tm - pArticleDownloader->GetLastUpdateTime() > g_pOptions->GetTerminateTimeout() &&
pArticleDownloader->GetStatus() == ArticleDownloader::adRunning)
{
ArticleInfo* pArticleInfo = pArticleDownloader->GetArticleInfo();
debug("Terminating hanging download %s", pArticleDownloader->GetInfoName());
if (pArticleDownloader->Terminate())
{
error("Terminated hanging download %s @ %s", pArticleDownloader->GetInfoName(),
pArticleDownloader->GetConnectionName());
pArticleInfo->SetStatus(ArticleInfo::aiUndefined);
}
else
{
error("Could not terminate hanging download %s @ %s", pArticleDownloader->GetInfoName(),
pArticleDownloader->GetConnectionName());
}
m_ActiveDownloads.erase(it);
pArticleDownloader->GetFileInfo()->SetActiveDownloads(pArticleDownloader->GetFileInfo()->GetActiveDownloads() - 1);
pArticleDownloader->GetFileInfo()->GetNZBInfo()->SetActiveDownloads(pArticleDownloader->GetFileInfo()->GetNZBInfo()->GetActiveDownloads() - 1);
pArticleDownloader->GetFileInfo()->GetNZBInfo()->SetDownloadedSize(pArticleDownloader->GetFileInfo()->GetNZBInfo()->GetDownloadedSize() + pArticleDownloader->GetDownloadedSize());
// it's not safe to destroy pArticleDownloader, because the state of object is unknown
delete pArticleDownloader;
it = m_ActiveDownloads.begin();
continue;
}
it++;
}
DownloadQueue::Unlock();
}
ParChecker::EFileStatus ParCoordinator::PostParChecker::FindFileCrc(const char* szFilename,
unsigned long* lCrc, SegmentList* pSegments)
{
CompletedFile* pCompletedFile = NULL;
for (CompletedFiles::iterator it = m_pPostInfo->GetNZBInfo()->GetCompletedFiles()->begin(); it != m_pPostInfo->GetNZBInfo()->GetCompletedFiles()->end(); it++)
{
CompletedFile* pCompletedFile2 = *it;
if (!strcasecmp(pCompletedFile2->GetFileName(), szFilename))
{
pCompletedFile = pCompletedFile2;
break;
}
}
if (!pCompletedFile)
{
return ParChecker::fsUnknown;
}
debug("Found completed file: %s, CRC: %.8x, Status: %i", Util::BaseFileName(pCompletedFile->GetFileName()), pCompletedFile->GetCrc(), (int)pCompletedFile->GetStatus());
*lCrc = pCompletedFile->GetCrc();
if (pCompletedFile->GetStatus() == CompletedFile::cfPartial && pCompletedFile->GetID() > 0 &&
!m_pPostInfo->GetNZBInfo()->GetReprocess())
{
FileInfo* pTmpFileInfo = new FileInfo(pCompletedFile->GetID());
if (!g_pDiskState->LoadFileState(pTmpFileInfo, NULL, true))
{
delete pTmpFileInfo;
return ParChecker::fsUnknown;
}
for (FileInfo::Articles::iterator it = pTmpFileInfo->GetArticles()->begin(); it != pTmpFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pa = *it;
ParChecker::Segment* pSegment = new Segment(pa->GetStatus() == ArticleInfo::aiFinished,
pa->GetSegmentOffset(), pa->GetSegmentSize(), pa->GetCrc());
pSegments->push_back(pSegment);
}
delete pTmpFileInfo;
}
return pCompletedFile->GetStatus() == CompletedFile::cfSuccess ? ParChecker::fsSuccess :
pCompletedFile->GetStatus() == CompletedFile::cfFailure &&
!m_pPostInfo->GetNZBInfo()->GetReprocess() ? ParChecker::fsFailure :
pCompletedFile->GetStatus() == CompletedFile::cfPartial && pSegments->size() > 0 &&
!m_pPostInfo->GetNZBInfo()->GetReprocess()? ParChecker::fsPartial :
ParChecker::fsUnknown;
}
void NZBFile::CalcHashes()
{
TempFileList fileList;
for (FileList::iterator it = m_pNZBInfo->GetFileList()->begin(); it != m_pNZBInfo->GetFileList()->end(); it++)
{
fileList.push_back(*it);
}
fileList.sort(CompareFileInfo);
unsigned int iFullContentHash = 0;
unsigned int iFilteredContentHash = 0;
int iUseForFilteredCount = 0;
for (TempFileList::iterator it = fileList.begin(); it != fileList.end(); it++)
{
FileInfo* pFileInfo = *it;
// check file extension
bool bSkip = !pFileInfo->GetParFile() &&
Util::MatchFileExt(pFileInfo->GetFilename(), g_pOptions->GetExtCleanupDisk(), ",;");
for (FileInfo::Articles::iterator it = pFileInfo->GetArticles()->begin(); it != pFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pArticle = *it;
int iLen = strlen(pArticle->GetMessageID());
iFullContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFullContentHash);
if (!bSkip)
{
iFilteredContentHash = Util::HashBJ96(pArticle->GetMessageID(), iLen, iFilteredContentHash);
iUseForFilteredCount++;
}
}
}
// if filtered hash is based on less than a half of files - do not use filtered hash at all
if (iUseForFilteredCount < (int)fileList.size() / 2)
{
iFilteredContentHash = 0;
}
m_pNZBInfo->SetFullContentHash(iFullContentHash);
m_pNZBInfo->SetFilteredContentHash(iFilteredContentHash);
}
void QueueCoordinator::DiscardDiskFile(FileInfo* pFileInfo)
{
// deleting temporary files
if (!g_pOptions->GetDirectWrite())
{
for (FileInfo::Articles::iterator it = pFileInfo->GetArticles()->begin(); it != pFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pa = *it;
if (pa->GetResultFilename())
{
remove(pa->GetResultFilename());
}
}
}
if (g_pOptions->GetDirectWrite() && pFileInfo->GetOutputFilename())
{
remove(pFileInfo->GetOutputFilename());
}
}
void ArticleDownloader::CompleteFileParts()
{
debug("Completing file parts");
debug("ArticleFilename: %s", m_pFileInfo->GetFilename());
SetStatus(adJoining);
bool bDirectWrite = g_pOptions->GetDirectWrite() && m_pFileInfo->GetOutputInitialized();
char szNZBName[1024];
char szNZBDestDir[1024];
// the locking is needed for accessing the memebers of NZBInfo
g_pDownloadQueueHolder->LockQueue();
strncpy(szNZBName, m_pFileInfo->GetNZBInfo()->GetName(), 1024);
strncpy(szNZBDestDir, m_pFileInfo->GetNZBInfo()->GetDestDir(), 1024);
g_pDownloadQueueHolder->UnlockQueue();
szNZBName[1024-1] = '\0';
szNZBDestDir[1024-1] = '\0';
char InfoFilename[1024];
snprintf(InfoFilename, 1024, "%s%c%s", szNZBName, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename());
InfoFilename[1024-1] = '\0';
if (!g_pOptions->GetDecode())
{
detail("Moving articles for %s", InfoFilename);
}
else if (bDirectWrite)
{
detail("Checking articles for %s", InfoFilename);
}
else
{
detail("Joining articles for %s", InfoFilename);
}
// Ensure the DstDir is created
char szErrBuf[1024];
if (!Util::ForceDirectories(szNZBDestDir, szErrBuf, sizeof(szErrBuf)))
{
error("Could not create directory %s: %s", szNZBDestDir, szErrBuf);
SetStatus(adJoined);
return;
}
char ofn[1024];
snprintf(ofn, 1024, "%s%c%s", szNZBDestDir, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename());
ofn[1024-1] = '\0';
// prevent overwriting existing files
int dupcount = 0;
while (Util::FileExists(ofn))
{
dupcount++;
snprintf(ofn, 1024, "%s%c%s_duplicate%d", szNZBDestDir, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename(), dupcount);
ofn[1024-1] = '\0';
}
FILE* outfile = NULL;
char tmpdestfile[1024];
snprintf(tmpdestfile, 1024, "%s.tmp", ofn);
tmpdestfile[1024-1] = '\0';
if (g_pOptions->GetDecode() && !bDirectWrite)
{
remove(tmpdestfile);
outfile = fopen(tmpdestfile, "wb+");
if (!outfile)
{
error("Could not create file %s!", tmpdestfile);
SetStatus(adJoined);
return;
}
if (g_pOptions->GetWriteBufferSize() == -1 && (*m_pFileInfo->GetArticles())[0])
{
setvbuf(outfile, (char *)NULL, _IOFBF, (*m_pFileInfo->GetArticles())[0]->GetSize());
}
else if (g_pOptions->GetWriteBufferSize() > 0)
{
setvbuf(outfile, (char *)NULL, _IOFBF, g_pOptions->GetWriteBufferSize());
}
}
else if (!g_pOptions->GetDecode())
{
remove(tmpdestfile);
if (!Util::CreateDirectory(ofn))
{
error("Could not create directory %s! Errcode: %i", ofn, errno);
SetStatus(adJoined);
return;
}
}
bool complete = true;
int iBrokenCount = 0;
static const int BUFFER_SIZE = 1024 * 50;
char* buffer = NULL;
if (g_pOptions->GetDecode() && !bDirectWrite)
{
buffer = (char*)malloc(BUFFER_SIZE);
}
for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pa = *it;
if (pa->GetStatus() != ArticleInfo::aiFinished)
{
iBrokenCount++;
complete = false;
}
else if (g_pOptions->GetDecode() && !bDirectWrite)
{
FILE* infile;
const char* fn = pa->GetResultFilename();
infile = fopen(fn, "rb");
if (infile)
{
int cnt = BUFFER_SIZE;
while (cnt == BUFFER_SIZE)
{
cnt = (int)fread(buffer, 1, BUFFER_SIZE, infile);
fwrite(buffer, 1, cnt, outfile);
SetLastUpdateTimeNow();
}
fclose(infile);
}
else
{
complete = false;
iBrokenCount++;
detail("Could not find file %s. Status is broken", fn);
}
}
else if (!g_pOptions->GetDecode())
{
const char* fn = pa->GetResultFilename();
char dstFileName[1024];
snprintf(dstFileName, 1024, "%s%c%03i", ofn, (int)PATH_SEPARATOR, pa->GetPartNumber());
dstFileName[1024-1] = '\0';
if (!Util::MoveFile(fn, dstFileName))
{
error("Could not move file %s to %s! Errcode: %i", fn, dstFileName, errno);
}
}
}
if (buffer)
{
free(buffer);
}
if (outfile)
{
fclose(outfile);
if (!Util::MoveFile(tmpdestfile, ofn))
{
error("Could not move file %s to %s! Errcode: %i", tmpdestfile, ofn, errno);
}
}
if (bDirectWrite)
{
if (!Util::MoveFile(m_szOutputFilename, ofn))
{
error("Could not move file %s to %s! Errcode: %i", m_szOutputFilename, ofn, errno);
}
// if destination directory was changed delete the old directory (if empty)
int iLen = strlen(szNZBDestDir);
if (!(!strncmp(szNZBDestDir, m_szOutputFilename, iLen) &&
(m_szOutputFilename[iLen] == PATH_SEPARATOR || m_szOutputFilename[iLen] == ALT_PATH_SEPARATOR)))
{
debug("Checking old dir for: %s", m_szOutputFilename);
char szOldDestDir[1024];
int iMaxlen = Util::BaseFileName(m_szOutputFilename) - m_szOutputFilename;
if (iMaxlen > 1024-1) iMaxlen = 1024-1;
strncpy(szOldDestDir, m_szOutputFilename, iMaxlen);
szOldDestDir[iMaxlen] = '\0';
if (Util::DirEmpty(szOldDestDir))
{
debug("Deleting old dir: %s", szOldDestDir);
rmdir(szOldDestDir);
}
}
}
if (!bDirectWrite || g_pOptions->GetContinuePartial())
{
for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++)
{
ArticleInfo* pa = *it;
remove(pa->GetResultFilename());
}
}
if (complete)
{
info("Successfully downloaded %s", InfoFilename);
}
else
{
warn("%i of %i article downloads failed for \"%s\"", iBrokenCount, m_pFileInfo->GetArticles()->size(), InfoFilename);
if (g_pOptions->GetCreateBrokenLog())
{
char szBrokenLogName[1024];
snprintf(szBrokenLogName, 1024, "%s%c_brokenlog.txt", szNZBDestDir, (int)PATH_SEPARATOR);
szBrokenLogName[1024-1] = '\0';
FILE* file = fopen(szBrokenLogName, "ab");
fprintf(file, "%s (%i/%i)%s", m_pFileInfo->GetFilename(), m_pFileInfo->GetArticles()->size() - iBrokenCount, m_pFileInfo->GetArticles()->size(), LINE_ENDING);
fclose(file);
}
}
// the locking is needed for accessing the memebers of NZBInfo
g_pDownloadQueueHolder->LockQueue();
m_pFileInfo->GetNZBInfo()->GetCompletedFiles()->push_back(strdup(ofn));
if (strcmp(m_pFileInfo->GetNZBInfo()->GetDestDir(), szNZBDestDir))
{
// destination directory was changed during completion, need to move the file
MoveCompletedFiles(m_pFileInfo->GetNZBInfo(), szNZBDestDir);
}
g_pDownloadQueueHolder->UnlockQueue();
SetStatus(adJoined);
}
bool NZBFile::ParseNZB(IUnknown* nzb)
{
MSXML::IXMLDOMDocumentPtr doc = nzb;
MSXML::IXMLDOMNodePtr root = doc->documentElement;
MSXML::IXMLDOMNodePtr node = root->selectSingleNode("/nzb/head/meta[@type='password']");
if (node)
{
_bstr_t password(node->Gettext());
m_szPassword = strdup(password);
}
MSXML::IXMLDOMNodeListPtr fileList = root->selectNodes("/nzb/file");
for (int i = 0; i < fileList->Getlength(); i++)
{
node = fileList->Getitem(i);
MSXML::IXMLDOMNodePtr attribute = node->Getattributes()->getNamedItem("subject");
if (!attribute) return false;
_bstr_t subject(attribute->Gettext());
FileInfo* pFileInfo = new FileInfo();
pFileInfo->SetSubject(subject);
attribute = node->Getattributes()->getNamedItem("date");
if (attribute)
{
_bstr_t date(attribute->Gettext());
pFileInfo->SetTime(atoi(date));
}
MSXML::IXMLDOMNodeListPtr groupList = node->selectNodes("groups/group");
for (int g = 0; g < groupList->Getlength(); g++)
{
MSXML::IXMLDOMNodePtr node = groupList->Getitem(g);
_bstr_t group = node->Gettext();
pFileInfo->GetGroups()->push_back(strdup((const char*)group));
}
MSXML::IXMLDOMNodeListPtr segmentList = node->selectNodes("segments/segment");
for (int g = 0; g < segmentList->Getlength(); g++)
{
MSXML::IXMLDOMNodePtr node = segmentList->Getitem(g);
_bstr_t id = node->Gettext();
char szId[2048];
snprintf(szId, 2048, "<%s>", (const char*)id);
MSXML::IXMLDOMNodePtr attribute = node->Getattributes()->getNamedItem("number");
if (!attribute) return false;
_bstr_t number(attribute->Gettext());
attribute = node->Getattributes()->getNamedItem("bytes");
if (!attribute) return false;
_bstr_t bytes(attribute->Gettext());
int partNumber = atoi(number);
int lsize = atoi(bytes);
if (partNumber > 0)
{
ArticleInfo* pArticle = new ArticleInfo();
pArticle->SetPartNumber(partNumber);
pArticle->SetMessageID(szId);
pArticle->SetSize(lsize);
AddArticle(pFileInfo, pArticle);
}
}
AddFileInfo(pFileInfo);
}
return true;
}
void QueueCoordinator::ArticleCompleted(ArticleDownloader* pArticleDownloader)
{
debug("Article downloaded");
FileInfo* pFileInfo = pArticleDownloader->GetFileInfo();
NZBInfo* pNZBInfo = pFileInfo->GetNZBInfo();
ArticleInfo* pArticleInfo = pArticleDownloader->GetArticleInfo();
bool bRetry = false;
bool fileCompleted = false;
DownloadQueue* pDownloadQueue = DownloadQueue::Lock();
if (pArticleDownloader->GetStatus() == ArticleDownloader::adFinished)
{
pArticleInfo->SetStatus(ArticleInfo::aiFinished);
pFileInfo->SetSuccessSize(pFileInfo->GetSuccessSize() + pArticleInfo->GetSize());
pNZBInfo->SetCurrentSuccessSize(pNZBInfo->GetCurrentSuccessSize() + pArticleInfo->GetSize());
pNZBInfo->SetParCurrentSuccessSize(pNZBInfo->GetParCurrentSuccessSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0));
pFileInfo->SetSuccessArticles(pFileInfo->GetSuccessArticles() + 1);
pNZBInfo->SetCurrentSuccessArticles(pNZBInfo->GetCurrentSuccessArticles() + 1);
}
else if (pArticleDownloader->GetStatus() == ArticleDownloader::adFailed)
{
pArticleInfo->SetStatus(ArticleInfo::aiFailed);
pFileInfo->SetFailedSize(pFileInfo->GetFailedSize() + pArticleInfo->GetSize());
pNZBInfo->SetCurrentFailedSize(pNZBInfo->GetCurrentFailedSize() + pArticleInfo->GetSize());
pNZBInfo->SetParCurrentFailedSize(pNZBInfo->GetParCurrentFailedSize() + (pFileInfo->GetParFile() ? pArticleInfo->GetSize() : 0));
pFileInfo->SetFailedArticles(pFileInfo->GetFailedArticles() + 1);
pNZBInfo->SetCurrentFailedArticles(pNZBInfo->GetCurrentFailedArticles() + 1);
}
else if (pArticleDownloader->GetStatus() == ArticleDownloader::adRetry)
{
pArticleInfo->SetStatus(ArticleInfo::aiUndefined);
bRetry = true;
}
if (!bRetry)
{
pFileInfo->SetRemainingSize(pFileInfo->GetRemainingSize() - pArticleInfo->GetSize());
pNZBInfo->SetRemainingSize(pNZBInfo->GetRemainingSize() - pArticleInfo->GetSize());
if (pFileInfo->GetPaused())
{
pNZBInfo->SetPausedSize(pNZBInfo->GetPausedSize() - pArticleInfo->GetSize());
}
pFileInfo->SetCompletedArticles(pFileInfo->GetCompletedArticles() + 1);
fileCompleted = (int)pFileInfo->GetArticles()->size() == pFileInfo->GetCompletedArticles();
pFileInfo->GetServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd);
pNZBInfo->GetCurrentServerStats()->ListOp(pArticleDownloader->GetServerStats(), ServerStatList::soAdd);
pFileInfo->SetPartialChanged(true);
}
if (!pFileInfo->GetFilenameConfirmed() &&
pArticleDownloader->GetStatus() == ArticleDownloader::adFinished &&
pArticleDownloader->GetArticleFilename())
{
pFileInfo->SetFilename(pArticleDownloader->GetArticleFilename());
pFileInfo->SetFilenameConfirmed(true);
if (g_pOptions->GetDupeCheck() &&
pNZBInfo->GetDupeMode() != dmForce &&
!pNZBInfo->GetManyDupeFiles() &&
Util::FileExists(pNZBInfo->GetDestDir(), pFileInfo->GetFilename()))
{
warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", pFileInfo->GetFilename());
fileCompleted = false;
pFileInfo->SetAutoDeleted(true);
DeleteQueueEntry(pDownloadQueue, pFileInfo);
}
}
pNZBInfo->SetDownloadedSize(pNZBInfo->GetDownloadedSize() + pArticleDownloader->GetDownloadedSize());
bool deleteFileObj = false;
if (fileCompleted && !pFileInfo->GetDeleted())
{
// all jobs done
DownloadQueue::Unlock();
pArticleDownloader->CompleteFileParts();
pDownloadQueue = DownloadQueue::Lock();
deleteFileObj = true;
}
CheckHealth(pDownloadQueue, pFileInfo);
bool hasOtherDownloaders = false;
for (ActiveDownloads::iterator it = m_ActiveDownloads.begin(); it != m_ActiveDownloads.end(); it++)
{
ArticleDownloader* pDownloader = *it;
if (pDownloader != pArticleDownloader && pDownloader->GetFileInfo() == pFileInfo)
{
hasOtherDownloaders = true;
break;
}
}
deleteFileObj |= pFileInfo->GetDeleted() && !hasOtherDownloaders;
// remove downloader from downloader list
m_ActiveDownloads.erase(std::find(m_ActiveDownloads.begin(), m_ActiveDownloads.end(), pArticleDownloader));
pFileInfo->SetActiveDownloads(pFileInfo->GetActiveDownloads() - 1);
pNZBInfo->SetActiveDownloads(pNZBInfo->GetActiveDownloads() - 1);
if (deleteFileObj)
{
DeleteFileInfo(pDownloadQueue, pFileInfo, fileCompleted);
pDownloadQueue->Save();
}
DownloadQueue::Unlock();
}
