void ArticleDownloader::CompleteFileParts() { debug("Completing file parts"); debug("ArticleFilename: %s", m_pFileInfo->GetFilename()); SetStatus(adJoining); bool bDirectWrite = g_pOptions->GetDirectWrite() && m_pFileInfo->GetOutputInitialized(); char szNZBName[1024]; char szNZBDestDir[1024]; // the locking is needed for accessing the memebers of NZBInfo g_pDownloadQueueHolder->LockQueue(); strncpy(szNZBName, m_pFileInfo->GetNZBInfo()->GetName(), 1024); strncpy(szNZBDestDir, m_pFileInfo->GetNZBInfo()->GetDestDir(), 1024); g_pDownloadQueueHolder->UnlockQueue(); szNZBName[1024-1] = '\0'; szNZBDestDir[1024-1] = '\0'; char InfoFilename[1024]; snprintf(InfoFilename, 1024, "%s%c%s", szNZBName, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename()); InfoFilename[1024-1] = '\0'; if (!g_pOptions->GetDecode()) { detail("Moving articles for %s", InfoFilename); } else if (bDirectWrite) { detail("Checking articles for %s", InfoFilename); } else { detail("Joining articles for %s", InfoFilename); } // Ensure the DstDir is created char szErrBuf[1024]; if (!Util::ForceDirectories(szNZBDestDir, szErrBuf, sizeof(szErrBuf))) { error("Could not create directory %s: %s", szNZBDestDir, szErrBuf); SetStatus(adJoined); return; } char ofn[1024]; snprintf(ofn, 1024, "%s%c%s", szNZBDestDir, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename()); ofn[1024-1] = '\0'; // prevent overwriting existing files int dupcount = 0; while (Util::FileExists(ofn)) { dupcount++; snprintf(ofn, 1024, "%s%c%s_duplicate%d", szNZBDestDir, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename(), dupcount); ofn[1024-1] = '\0'; } FILE* outfile = NULL; char tmpdestfile[1024]; snprintf(tmpdestfile, 1024, "%s.tmp", ofn); tmpdestfile[1024-1] = '\0'; if (g_pOptions->GetDecode() && !bDirectWrite) { remove(tmpdestfile); outfile = fopen(tmpdestfile, "wb+"); if (!outfile) { error("Could not create file %s!", tmpdestfile); SetStatus(adJoined); return; } if (g_pOptions->GetWriteBufferSize() == -1 && (*m_pFileInfo->GetArticles())[0]) { setvbuf(outfile, (char *)NULL, _IOFBF, (*m_pFileInfo->GetArticles())[0]->GetSize()); } else if (g_pOptions->GetWriteBufferSize() > 0) { setvbuf(outfile, (char *)NULL, _IOFBF, g_pOptions->GetWriteBufferSize()); } } else if (!g_pOptions->GetDecode()) { remove(tmpdestfile); if (!Util::CreateDirectory(ofn)) { error("Could not create directory %s! Errcode: %i", ofn, errno); SetStatus(adJoined); return; } } bool complete = true; int iBrokenCount = 0; static const int BUFFER_SIZE = 1024 * 50; char* buffer = NULL; if (g_pOptions->GetDecode() && !bDirectWrite) { buffer = (char*)malloc(BUFFER_SIZE); } for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++) { ArticleInfo* pa = *it; if (pa->GetStatus() != ArticleInfo::aiFinished) { iBrokenCount++; complete = false; } else if (g_pOptions->GetDecode() && !bDirectWrite) { FILE* infile; const char* fn = pa->GetResultFilename(); infile = fopen(fn, "rb"); if (infile) { int cnt = BUFFER_SIZE; while (cnt == BUFFER_SIZE) { cnt = (int)fread(buffer, 1, BUFFER_SIZE, infile); fwrite(buffer, 1, cnt, outfile); SetLastUpdateTimeNow(); } fclose(infile); } else { complete = false; iBrokenCount++; detail("Could not find file %s. Status is broken", fn); } } else if (!g_pOptions->GetDecode()) { const char* fn = pa->GetResultFilename(); char dstFileName[1024]; snprintf(dstFileName, 1024, "%s%c%03i", ofn, (int)PATH_SEPARATOR, pa->GetPartNumber()); dstFileName[1024-1] = '\0'; if (!Util::MoveFile(fn, dstFileName)) { error("Could not move file %s to %s! Errcode: %i", fn, dstFileName, errno); } } } if (buffer) { free(buffer); } if (outfile) { fclose(outfile); if (!Util::MoveFile(tmpdestfile, ofn)) { error("Could not move file %s to %s! Errcode: %i", tmpdestfile, ofn, errno); } } if (bDirectWrite) { if (!Util::MoveFile(m_szOutputFilename, ofn)) { error("Could not move file %s to %s! Errcode: %i", m_szOutputFilename, ofn, errno); } // if destination directory was changed delete the old directory (if empty) int iLen = strlen(szNZBDestDir); if (!(!strncmp(szNZBDestDir, m_szOutputFilename, iLen) && (m_szOutputFilename[iLen] == PATH_SEPARATOR || m_szOutputFilename[iLen] == ALT_PATH_SEPARATOR))) { debug("Checking old dir for: %s", m_szOutputFilename); char szOldDestDir[1024]; int iMaxlen = Util::BaseFileName(m_szOutputFilename) - m_szOutputFilename; if (iMaxlen > 1024-1) iMaxlen = 1024-1; strncpy(szOldDestDir, m_szOutputFilename, iMaxlen); szOldDestDir[iMaxlen] = '\0'; if (Util::DirEmpty(szOldDestDir)) { debug("Deleting old dir: %s", szOldDestDir); rmdir(szOldDestDir); } } } if (!bDirectWrite || g_pOptions->GetContinuePartial()) { for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++) { ArticleInfo* pa = *it; remove(pa->GetResultFilename()); } } if (complete) { info("Successfully downloaded %s", InfoFilename); } else { warn("%i of %i article downloads failed for \"%s\"", iBrokenCount, m_pFileInfo->GetArticles()->size(), InfoFilename); if (g_pOptions->GetCreateBrokenLog()) { char szBrokenLogName[1024]; snprintf(szBrokenLogName, 1024, "%s%c_brokenlog.txt", szNZBDestDir, (int)PATH_SEPARATOR); szBrokenLogName[1024-1] = '\0'; FILE* file = fopen(szBrokenLogName, "ab"); fprintf(file, "%s (%i/%i)%s", m_pFileInfo->GetFilename(), m_pFileInfo->GetArticles()->size() - iBrokenCount, m_pFileInfo->GetArticles()->size(), LINE_ENDING); fclose(file); } } // the locking is needed for accessing the memebers of NZBInfo g_pDownloadQueueHolder->LockQueue(); m_pFileInfo->GetNZBInfo()->GetCompletedFiles()->push_back(strdup(ofn)); if (strcmp(m_pFileInfo->GetNZBInfo()->GetDestDir(), szNZBDestDir)) { // destination directory was changed during completion, need to move the file MoveCompletedFiles(m_pFileInfo->GetNZBInfo(), szNZBDestDir); } g_pDownloadQueueHolder->UnlockQueue(); SetStatus(adJoined); }
void ArticleWriter::CompleteFileParts() { debug("Completing file parts"); debug("ArticleFilename: %s", m_fileInfo->GetFilename()); bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_fileInfo->GetOutputInitialized(); BString<1024> nzbName; BString<1024> nzbDestDir; BString<1024> filename; { GuardedDownloadQueue guard = DownloadQueue::Guard(); nzbName = m_fileInfo->GetNzbInfo()->GetName(); nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir(); filename = m_fileInfo->GetFilename(); } BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename); bool cached = m_fileInfo->GetCachedArticles() > 0; if (g_Options->GetRawArticle()) { detail("Moving articles for %s", *infoFilename); } else if (directWrite && cached) { detail("Writing articles for %s", *infoFilename); } else if (directWrite) { detail("Checking articles for %s", *infoFilename); } else { detail("Joining articles for %s", *infoFilename); } // Ensure the DstDir is created CString errmsg; if (!FileSystem::ForceDirectories(nzbDestDir, errmsg)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not create directory %s: %s", *nzbDestDir, *errmsg); return; } CString ofn; if (m_fileInfo->GetForceDirectWrite()) { ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename); } else { ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename); } DiskFile outfile; BString<1024> tmpdestfile("%s.tmp", *ofn); if (!g_Options->GetRawArticle() && !directWrite) { FileSystem::DeleteFile(tmpdestfile); if (!outfile.Open(tmpdestfile, DiskFile::omWrite)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not create file %s: %s", *tmpdestfile, *FileSystem::GetLastErrorMessage()); return; } } else if (directWrite && cached) { if (!outfile.Open(m_outputFilename, DiskFile::omReadWrite)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not open file %s: %s", *m_outputFilename, *FileSystem::GetLastErrorMessage()); return; } tmpdestfile = *m_outputFilename; } else if (g_Options->GetRawArticle()) { FileSystem::DeleteFile(tmpdestfile); if (!FileSystem::CreateDirectory(ofn)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not create directory %s: %s", *ofn, *FileSystem::GetLastErrorMessage()); return; } } if (outfile.Active()) { SetWriteBuffer(outfile, 0); } uint32 crc = 0; { std::unique_ptr<ArticleCache::FlushGuard> flushGuard; if (cached) { flushGuard = std::make_unique<ArticleCache::FlushGuard>(g_ArticleCache->GuardFlush()); } CharBuffer buffer; bool firstArticle = true; if (!g_Options->GetRawArticle() && !directWrite) { buffer.Reserve(1024 * 64); } for (ArticleInfo* pa : m_fileInfo->GetArticles()) { if (pa->GetStatus() != ArticleInfo::aiFinished) { continue; } if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 && pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1) { memset(buffer, 0, buffer.Size()); if (!g_Options->GetSkipWrite()) { while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 && outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size()))); } } if (pa->GetSegmentContent()) { if (!g_Options->GetSkipWrite()) { outfile.Seek(pa->GetSegmentOffset()); outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize()); } pa->DiscardSegment(); } else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite()) { DiskFile infile; if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead)) { int cnt = buffer.Size(); while (cnt == buffer.Size()) { cnt = (int)infile.Read(buffer, buffer.Size()); outfile.Write(buffer, cnt); } infile.Close(); } else { m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1); m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1); m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not find file %s for %s [%i/%i]", pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(), (int)m_fileInfo->GetArticles()->size()); } } else if (g_Options->GetRawArticle()) { BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber()); if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not move file %s to %s: %s", pa->GetResultFilename(), *dstFileName, *FileSystem::GetLastErrorMessage()); } } if (m_format == Decoder::efYenc) { crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize()); firstArticle = false; } } buffer.Clear(); } if (outfile.Active()) { outfile.Close(); if (!directWrite && !FileSystem::MoveFile(tmpdestfile, ofn)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not move file %s to %s: %s", *tmpdestfile, *ofn, *FileSystem::GetLastErrorMessage()); } } if (directWrite) { if (!FileSystem::SameFilename(m_outputFilename, ofn) && !FileSystem::MoveFile(m_outputFilename, ofn)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not move file %s to %s: %s", *m_outputFilename, *ofn, *FileSystem::GetLastErrorMessage()); } // if destination directory was changed delete the old directory (if empty) int len = strlen(nzbDestDir); if (!(!strncmp(nzbDestDir, m_outputFilename, len) && (m_outputFilename[len] == PATH_SEPARATOR || m_outputFilename[len] == ALT_PATH_SEPARATOR))) { debug("Checking old dir for: %s", *m_outputFilename); BString<1024> oldDestDir; oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename)); if (FileSystem::DirEmpty(oldDestDir)) { debug("Deleting old dir: %s", *oldDestDir); FileSystem::RemoveDirectory(oldDestDir); } } } if (!directWrite) { for (ArticleInfo* pa : m_fileInfo->GetArticles()) { if (pa->GetResultFilename()) { FileSystem::DeleteFile(pa->GetResultFilename()); } } } if (m_fileInfo->GetTotalArticles() == m_fileInfo->GetSuccessArticles()) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Successfully downloaded %s", *infoFilename); } else if (m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles() > 0) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkWarning, "%i of %i article downloads failed for \"%s\"", m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(), m_fileInfo->GetTotalArticles(), *infoFilename); } else { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename); } { GuardedDownloadQueue guard = DownloadQueue::Guard(); m_fileInfo->SetCrc(crc); m_fileInfo->SetOutputFilename(ofn); if (strcmp(m_fileInfo->GetFilename(), filename)) { // file was renamed during completion, need to move the file ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename()); if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn)) { m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(), *ofn, *FileSystem::GetLastErrorMessage()); } m_fileInfo->SetOutputFilename(ofn); } if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir)) { // destination directory was changed during completion, need to move the file MoveCompletedFiles(m_fileInfo->GetNzbInfo(), nzbDestDir); } } }