예제 #1
0
bool FileSystem::LoadFileIntoBuffer(const char* filename, CharBuffer& buffer, bool addTrailingNull)
{
	DiskFile file;
	if (!file.Open(filename, DiskFile::omRead))
	{
		return false;
	}

	// obtain file size.
	file.Seek(0, DiskFile::soEnd);
	int size  = (int)file.Position();
	file.Seek(0);

	// allocate memory to contain the whole file.
	buffer.Reserve(size + (addTrailingNull ? 1 : 0));

	// copy the file into the buffer.
	file.Read(buffer, size);
	file.Close();

	if (addTrailingNull)
	{
		buffer[size] = 0;
	}

	return true;
}
예제 #2
0
bool RarVolume::Read()
{
	debug("Checking file %s", *m_filename);

	DiskFile file;
	if (!file.Open(m_filename, DiskFile::omRead))
	{
		return false;
	}

	m_version = DetectRarVersion(file);
	file.Seek(0);

	bool ok = false;

	switch (m_version)
	{
		case 3:
			ok = ReadRar3Volume(file);
			break;

		case 5:
			ok = ReadRar5Volume(file);
			break;
	}

	file.Close();
	DecryptFree();

	LogDebugInfo();

	return ok;
}
예제 #3
0
void NntpProcessor::SendSegment()
{
	detail("[%i] Sending segment %s (%i=%lli:%i)", m_id, *m_filename, m_part, (long long)m_offset, m_size);

	if (m_speed > 0)
	{
		m_start = Util::GetCurrentTicks();
	}

	BString<1024> fullFilename("%s/%s", m_dataDir, *m_filename);
	BString<1024> cacheFileDir("%s/%s", m_cacheDir, *m_filename);
	BString<1024> cacheFileName("%i=%lli-%i", m_part, (long long)m_offset, m_size);
	BString<1024> cacheFullFilename("%s/%s", *cacheFileDir, *cacheFileName);
	BString<1024> cacheKey("%s/%s", *m_filename, *cacheFileName);

	const char* cachedData = nullptr;
	int cachedSize;
	if (m_cache)
	{
		m_cache->Find(cacheKey, cachedData, cachedSize);
	}

	DiskFile cacheFile;
	bool readCache = !cachedData && m_cacheDir && cacheFile.Open(cacheFullFilename, DiskFile::omRead);
	bool writeCache = !cachedData && m_cacheDir && !readCache;
	StringBuilder cacheMem;
	if (m_cache && !cachedData)
	{
		cacheMem.Reserve((int)(m_size * 1.1));
	}

	CString errmsg;
	if (writeCache && !FileSystem::ForceDirectories(cacheFileDir, errmsg))
	{
		error("Could not create directory %s: %s", *cacheFileDir, *errmsg);
	}

	if (writeCache && !cacheFile.Open(cacheFullFilename, DiskFile::omWrite))
	{
		error("Could not create file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
	}

	if (!cachedData && !readCache && !FileSystem::FileExists(fullFilename))
	{
		m_connection->WriteLine(CString::FormatStr("430 Article not found\r\n"));
		return;
	}

	YEncoder encoder(fullFilename, m_part, m_offset, m_size, 
		[proc = this, writeCache, &cacheFile, &cacheMem](const char* buf, int size)
		{
			if (proc->m_cache)
			{
				cacheMem.Append(buf);
			}
			if (writeCache)
			{
				cacheFile.Write(buf, size);
			}
			proc->SendData(buf, size);
		});

	if (!cachedData && !readCache && !encoder.OpenFile(errmsg))
	{
		m_connection->WriteLine(CString::FormatStr("403 %s\r\n", *errmsg));
		return;
	}

	m_connection->WriteLine(CString::FormatStr("%i, 0 %s\r\n", m_sendHeaders ? 222 : 220, m_messageid));
	if (m_sendHeaders)
	{
		m_connection->WriteLine(CString::FormatStr("Message-ID: %s\r\n", m_messageid));
		m_connection->WriteLine(CString::FormatStr("Subject: \"%s\"\r\n", FileSystem::BaseFileName(m_filename)));
		m_connection->WriteLine("\r\n");
	}

	if (cachedData)
	{
		SendData(cachedData, cachedSize);
	}
	else if (readCache)
	{
		cacheFile.Seek(0, DiskFile::soEnd);
		int size = (int)cacheFile.Position();
		CharBuffer buf(size);
		cacheFile.Seek(0);
		if (cacheFile.Read((char*)buf, size) != size)
		{
			error("Could not read file %s: %s", *cacheFullFilename, *FileSystem::GetLastErrorMessage());
		}
		if (m_cache)
		{
			cacheMem.Append(buf, size);
		}
		SendData(buf, size);
	}
	else
	{
		encoder.WriteSegment();
	}

	if (!cachedData && cacheMem.Length() > 0)
	{
		m_cache->Append(cacheKey, cacheMem, cacheMem.Length());
	}

	m_connection->WriteLine(".\r\n");
}
예제 #4
0
bool RarVolume::ReadRar5Volume(DiskFile& file)
{
	debug("Reading rar5-file %s", *m_filename);

	file.Seek(8);

	while (!file.Eof())
	{
		RarBlock block = ReadRar5Block(file);
		if (!block.type)
		{
			return false;
		}

		if (block.type == RAR5_BLOCK_MAIN)
		{
			uint64 arcflags;
			if (!ReadV(file, &block, &arcflags)) return false;
			if (arcflags & RAR5_MAIN_VOLNR)
			{
				uint64 volnr;
				if (!ReadV(file, &block, &volnr)) return false;
				m_volumeNo = (uint32)volnr;
			}
			m_newNaming = true;
			m_multiVolume = (arcflags & RAR5_MAIN_ISVOL) != 0;
		}

		else if (block.type == RAR5_BLOCK_ENCRYPTION)
		{
			uint64 val;
			if (!ReadV(file, &block, &val)) return false;
			if (val != 0) return false; // supporting only AES
			if (!ReadV(file, &block, &val)) return false;
			uint8 kdfCount;
			uint8 salt[16];
			if (!Read(file, &block, &kdfCount, sizeof(kdfCount))) return false;
			if (!Read(file, &block, &salt, sizeof(salt))) return false;
			m_encrypted = true;
			if (m_password.Empty()) return false;
			if (!DecryptRar5Prepare(kdfCount, salt)) return false;
		}
		
		else if (block.type == RAR5_BLOCK_FILE)
		{
			RarFile innerFile;
			if (!ReadRar5File(file, block, innerFile)) return false;
			m_files.push_back(std::move(innerFile));
		}

		else if (block.type == RAR5_BLOCK_ENDARC)
		{
			uint64 endflags;
			if (!ReadV(file, &block, &endflags)) return false;
			m_hasNextVolume = (endflags & RAR5_ENDARC_NEXTVOL) != 0;
			break;
		}

		else if (block.type < 1 || block.type > 5)
		{
			// inlvaid block type
			return false;
		}

		uint64 skip = block.trailsize;
		if (m_encrypted)
		{
			skip -= 16 - m_decryptPos;
			if (m_decryptPos < 16)
			{
				skip += skip % 16 > 0 ? 16 - skip % 16 : 0;
				m_decryptPos = 16;
			}
			DecryptFree();
		}

		if (!file.Seek(skip, DiskFile::soCur))
		{
			return false;
		}
	}

	return true;
}
예제 #5
0
bool RarVolume::ReadRar3Volume(DiskFile& file)
{
	debug("Reading rar3-file %s", *m_filename);

	while (!file.Eof())
	{
		RarBlock block = ReadRar3Block(file);
		if (!block.type)
		{
			return false;
		}

		if (block.type == RAR3_BLOCK_MAIN)
		{
			if (block.flags & RAR3_MAIN_PASSWORD)
			{
				m_encrypted = true;
				if (m_password.Empty()) return false;
			}
			m_newNaming = block.flags & RAR3_MAIN_NEWNUMBERING;
			m_multiVolume = block.flags & RAR3_MAIN_VOLUME;
		}

		else if (block.type == RAR3_BLOCK_FILE)
		{
			RarFile innerFile;
			if (!ReadRar3File(file, block, innerFile)) return false;
			m_files.push_back(std::move(innerFile));
		}

		else if (block.type == RAR3_BLOCK_ENDARC)
		{
			if (block.flags & RAR3_ENDARC_DATACRC)
			{
				if (!Skip(file, &block, 4)) return false;
			}
			if (block.flags & RAR3_ENDARC_VOLNUMBER)
			{
				if (!Read32(file, &block, &m_volumeNo)) return false;
				m_hasNextVolume = (block.flags & RAR3_ENDARC_NEXTVOL) != 0;
			}
			break;
		}

		else if (block.type < 0x72 || block.type > 0x7b)
		{
			// inlvaid block type
			return false;
		}

		uint64 skip = block.trailsize;
		if (m_encrypted)
		{
			skip -= 16 - m_decryptPos;
			m_decryptPos = 16;
			DecryptFree();
		}

		if (!file.Seek(skip, DiskFile::soCur))
		{
			return false;
		}
	}

	return true;
}
예제 #6
0
void ArticleWriter::FlushCache()
{
	detail("Flushing cache for %s", *m_infoName);

	bool directWrite = g_Options->GetDirectWrite() && m_fileInfo->GetOutputInitialized();
	DiskFile outfile;
	bool needBufFile = false;
	int flushedArticles = 0;
	int64 flushedSize = 0;

	{
		ArticleCache::FlushGuard flushGuard = g_ArticleCache->GuardFlush();

		std::vector<ArticleInfo*> cachedArticles;

		{
			Guard contentGuard = g_ArticleCache->GuardContent();

			if (m_fileInfo->GetFlushLocked())
			{
				return;
			}

			m_fileInfo->SetFlushLocked(true);

			cachedArticles.reserve(m_fileInfo->GetArticles()->size());
			for (ArticleInfo* pa : m_fileInfo->GetArticles())
			{
				if (pa->GetSegmentContent())
				{
					cachedArticles.push_back(pa);
				}
			}
		}

		for (ArticleInfo* pa : cachedArticles)
		{
			if (m_fileInfo->GetDeleted() && !m_fileInfo->GetNzbInfo()->GetParking())
			{
				// the file was deleted during flushing: stop flushing immediately
				break;
			}

			if (directWrite && !outfile.Active())
			{
				if (!outfile.Open(m_fileInfo->GetOutputFilename(), DiskFile::omReadWrite))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not open file %s: %s", m_fileInfo->GetOutputFilename(),
						*FileSystem::GetLastErrorMessage());
					// prevent multiple error messages
					pa->DiscardSegment();
					flushedArticles++;
					break;
				}
				needBufFile = true;
			}

			BString<1024> destFile;

			if (!directWrite)
			{
				destFile.Format("%s.tmp", pa->GetResultFilename());
				if (!outfile.Open(destFile, DiskFile::omWrite))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not create file %s: %s", *destFile,
						*FileSystem::GetLastErrorMessage());
					// prevent multiple error messages
					pa->DiscardSegment();
					flushedArticles++;
					break;
				}
				needBufFile = true;
			}

			if (outfile.Active() && needBufFile)
			{
				SetWriteBuffer(outfile, 0);
				needBufFile = false;
			}

			if (directWrite)
			{
				outfile.Seek(pa->GetSegmentOffset());
			}

			if (!g_Options->GetSkipWrite())
			{
				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
			}

			flushedSize += pa->GetSegmentSize();
			flushedArticles++;

			pa->DiscardSegment();

			if (!directWrite)
			{
				outfile.Close();

				if (!FileSystem::MoveFile(destFile, pa->GetResultFilename()))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not rename file %s to %s: %s", *destFile, pa->GetResultFilename(),
						*FileSystem::GetLastErrorMessage());
				}
			}
		}

		outfile.Close();

		{
			Guard contentGuard = g_ArticleCache->GuardContent();
			m_fileInfo->SetCachedArticles(m_fileInfo->GetCachedArticles() - flushedArticles);
			m_fileInfo->SetFlushLocked(false);
		}
	}

	detail("Saved %i articles (%.2f MB) from cache into disk for %s", flushedArticles,
		(float)(flushedSize / 1024.0 / 1024.0), *m_infoName);
}
예제 #7
0
void ArticleWriter::CompleteFileParts()
{
	debug("Completing file parts");
	debug("ArticleFilename: %s", m_fileInfo->GetFilename());

	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_fileInfo->GetOutputInitialized();

	BString<1024> nzbName;
	BString<1024> nzbDestDir;
	BString<1024> filename;

	{
		GuardedDownloadQueue guard = DownloadQueue::Guard();
		nzbName = m_fileInfo->GetNzbInfo()->GetName();
		nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir();
		filename = m_fileInfo->GetFilename();
	}

	BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename);

	bool cached = m_fileInfo->GetCachedArticles() > 0;

	if (g_Options->GetRawArticle())
	{
		detail("Moving articles for %s", *infoFilename);
	}
	else if (directWrite && cached)
	{
		detail("Writing articles for %s", *infoFilename);
	}
	else if (directWrite)
	{
		detail("Checking articles for %s", *infoFilename);
	}
	else
	{
		detail("Joining articles for %s", *infoFilename);
	}

	// Ensure the DstDir is created
	CString errmsg;
	if (!FileSystem::ForceDirectories(nzbDestDir, errmsg))
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
			"Could not create directory %s: %s", *nzbDestDir, *errmsg);
		return;
	}

	CString ofn;
	if (m_fileInfo->GetForceDirectWrite())
	{
		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename);
	}
	else
	{
		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename);
	}

	DiskFile outfile;
	BString<1024> tmpdestfile("%s.tmp", *ofn);

	if (!g_Options->GetRawArticle() && !directWrite)
	{
		FileSystem::DeleteFile(tmpdestfile);
		if (!outfile.Open(tmpdestfile, DiskFile::omWrite))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not create file %s: %s", *tmpdestfile, *FileSystem::GetLastErrorMessage());
			return;
		}
	}
	else if (directWrite && cached)
	{
		if (!outfile.Open(m_outputFilename, DiskFile::omReadWrite))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not open file %s: %s", *m_outputFilename, *FileSystem::GetLastErrorMessage());
			return;
		}
		tmpdestfile = *m_outputFilename;
	}
	else if (g_Options->GetRawArticle())
	{
		FileSystem::DeleteFile(tmpdestfile);
		if (!FileSystem::CreateDirectory(ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not create directory %s: %s", *ofn, *FileSystem::GetLastErrorMessage());
			return;
		}
	}

	if (outfile.Active())
	{
		SetWriteBuffer(outfile, 0);
	}

	uint32 crc = 0;

	{
		std::unique_ptr<ArticleCache::FlushGuard> flushGuard;
		if (cached)
		{
			flushGuard = std::make_unique<ArticleCache::FlushGuard>(g_ArticleCache->GuardFlush());
		}

		CharBuffer buffer;
		bool firstArticle = true;

		if (!g_Options->GetRawArticle() && !directWrite)
		{
			buffer.Reserve(1024 * 64);
		}

		for (ArticleInfo* pa : m_fileInfo->GetArticles())
		{
			if (pa->GetStatus() != ArticleInfo::aiFinished)
			{
				continue;
			}

			if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 &&
				pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1)
			{
				memset(buffer, 0, buffer.Size());
				if (!g_Options->GetSkipWrite())
				{
					while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
						outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
				}
			}

			if (pa->GetSegmentContent())
			{
				if (!g_Options->GetSkipWrite())
				{
					outfile.Seek(pa->GetSegmentOffset());
					outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
				}
				pa->DiscardSegment();
			}
			else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite())
			{
				DiskFile infile;
				if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead))
				{
					int cnt = buffer.Size();
					while (cnt == buffer.Size())
					{
						cnt = (int)infile.Read(buffer, buffer.Size());
						outfile.Write(buffer, cnt);
					}
					infile.Close();
				}
				else
				{
					m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1);
					m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1);
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not find file %s for %s [%i/%i]",
						pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(),
						(int)m_fileInfo->GetArticles()->size());
				}
			}
			else if (g_Options->GetRawArticle())
			{
				BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber());
				if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not move file %s to %s: %s", pa->GetResultFilename(),
						*dstFileName, *FileSystem::GetLastErrorMessage());
				}
			}

			if (m_format == Decoder::efYenc)
			{
				crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
				firstArticle = false;
			}
		}

		buffer.Clear();
	}

	if (outfile.Active())
	{
		outfile.Close();
		if (!directWrite && !FileSystem::MoveFile(tmpdestfile, ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not move file %s to %s: %s", *tmpdestfile, *ofn,
				*FileSystem::GetLastErrorMessage());
		}
	}

	if (directWrite)
	{
		if (!FileSystem::SameFilename(m_outputFilename, ofn) &&
			!FileSystem::MoveFile(m_outputFilename, ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not move file %s to %s: %s", *m_outputFilename, *ofn,
				*FileSystem::GetLastErrorMessage());
		}

		// if destination directory was changed delete the old directory (if empty)
		int len = strlen(nzbDestDir);
		if (!(!strncmp(nzbDestDir, m_outputFilename, len) &&
			(m_outputFilename[len] == PATH_SEPARATOR || m_outputFilename[len] == ALT_PATH_SEPARATOR)))
		{
			debug("Checking old dir for: %s", *m_outputFilename);
			BString<1024> oldDestDir;
			oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
			if (FileSystem::DirEmpty(oldDestDir))
			{
				debug("Deleting old dir: %s", *oldDestDir);
				FileSystem::RemoveDirectory(oldDestDir);
			}
		}
	}

	if (!directWrite)
	{
		for (ArticleInfo* pa : m_fileInfo->GetArticles())
		{
			if (pa->GetResultFilename())
			{
				FileSystem::DeleteFile(pa->GetResultFilename());
			}
		}
	}

	if (m_fileInfo->GetTotalArticles() == m_fileInfo->GetSuccessArticles())
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Successfully downloaded %s", *infoFilename);
	}
	else if (m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles() > 0)
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
			"%i of %i article downloads failed for \"%s\"",
			m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(),
			m_fileInfo->GetTotalArticles(), *infoFilename);
	}
	else
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename);
	}

	{
		GuardedDownloadQueue guard = DownloadQueue::Guard();

		m_fileInfo->SetCrc(crc);
		m_fileInfo->SetOutputFilename(ofn);

		if (strcmp(m_fileInfo->GetFilename(), filename))
		{
			// file was renamed during completion, need to move the file
			ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
			if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn))
			{
				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
					"Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(),
					*ofn, *FileSystem::GetLastErrorMessage());
			}
			m_fileInfo->SetOutputFilename(ofn);
		}

		if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir))
		{
			// destination directory was changed during completion, need to move the file
			MoveCompletedFiles(m_fileInfo->GetNzbInfo(), nzbDestDir);
		}
	}
}