예제 #1
0
bool FileSystem::CopyFile(const char* srcFilename, const char* dstFilename)
{
	DiskFile infile;
	if (!infile.Open(srcFilename, DiskFile::omRead))
	{
		return false;
	}

	DiskFile outfile;
	if (!outfile.Open(dstFilename, DiskFile::omWrite))
	{
		return false;
	}

	CharBuffer buffer(1024 * 50);

	int cnt = buffer.Size();
	while (cnt == buffer.Size())
	{
		cnt = (int)infile.Read(buffer, buffer.Size());
		outfile.Write(buffer, cnt);
	}

	infile.Close();
	outfile.Close();

	return true;
}
예제 #2
0
bool RarVolume::Read()
{
	debug("Checking file %s", *m_filename);

	DiskFile file;
	if (!file.Open(m_filename, DiskFile::omRead))
	{
		return false;
	}

	m_version = DetectRarVersion(file);
	file.Seek(0);

	bool ok = false;

	switch (m_version)
	{
		case 3:
			ok = ReadRar3Volume(file);
			break;

		case 5:
			ok = ReadRar5Volume(file);
			break;
	}

	file.Close();
	DecryptFree();

	LogDebugInfo();

	return ok;
}
예제 #3
0
bool FileSystem::LoadFileIntoBuffer(const char* filename, CharBuffer& buffer, bool addTrailingNull)
{
	DiskFile file;
	if (!file.Open(filename, DiskFile::omRead))
	{
		return false;
	}

	// obtain file size.
	file.Seek(0, DiskFile::soEnd);
	int size  = (int)file.Position();
	file.Seek(0);

	// allocate memory to contain the whole file.
	buffer.Reserve(size + (addTrailingNull ? 1 : 0));

	// copy the file into the buffer.
	file.Read(buffer, size);
	file.Close();

	if (addTrailingNull)
	{
		buffer[size] = 0;
	}

	return true;
}
예제 #4
0
bool Par1Repairer::RemoveParFiles(void) {
	if (noiselevel > CommandLine::nlSilent
			&& parlist.size() > 0) {
			cout << endl << "Purge par files." << endl;
	}

	for (list<string>::const_iterator s=parlist.begin(); s!=parlist.end(); ++s) {
		DiskFile *diskfile = new DiskFile;

		if (diskfile->Open(*s)) {
			if (noiselevel > CommandLine::nlSilent) {
				string name;
				string path;
				DiskFile::SplitFilename((*s), path, name);
				cout << "Remove \"" << name << "\"." << endl;
			}

			if (diskfile->IsOpen())
				diskfile->Close();
			diskfile->Delete();
		}

		delete diskfile;
	}

	return true;
}
예제 #5
0
bool Copy(File& srcFile, const char* destFilename)
{
	// Operate in 16k buffers.
	const DWORD BUFFER_SIZE = 16 * 1024;

	DWORD fileSize = (DWORD)srcFile.GetLength();

	// See if the destination file exists.
	DiskFile destFile;
	if (!destFile.Open(destFilename, File::MODE_CREATE | File::MODE_WRITEONLY))
		return false;

	// Allocate the buffer space.
	BYTE* buffer = (BYTE*)_alloca(BUFFER_SIZE);

	// Keep copying until there is no more file left to copy.
	while (fileSize > 0)
	{
		// Copy the minimum of BUFFER_SIZE or the fileSize.
		DWORD readSize = min(BUFFER_SIZE, fileSize);
		srcFile.Read(buffer, readSize);
		destFile.Write(buffer, readSize);
		fileSize -= readSize;
	}

	// Close the destination virtual file.
	destFile.Close();

	return true;
}
예제 #6
0
// Delete all of the partly reconstructed files
bool Par1Repairer::DeleteIncompleteTargetFiles(void) {
	list<Par1RepairerSourceFile*>::iterator sf = verifylist.begin();

	// Iterate through each file in the verification list
	while (sf != verifylist.end()) {
		Par1RepairerSourceFile *sourcefile = *sf;
		if (sourcefile->GetTargetExists()) {
			DiskFile *targetfile = sourcefile->GetTargetFile();

			// Close and delete the file
			if (targetfile->IsOpen())
				targetfile->Close();
			targetfile->Delete();

			// Forget the file
			diskfilemap.Remove(targetfile);

			delete targetfile;

			// There is no target file
			sourcefile->SetTargetExists(false);
			sourcefile->SetTargetFile(0);
		}

		++sf;
	}

	return true;
}
예제 #7
0
bool Copy(File& srcFile, const char* destFilename)
{
	// Operate in 16k buffers.
	const uint32_t BUFFER_SIZE = 16 * 1024;

	uint32_t fileSize = (uint32_t)srcFile.GetLength();

	if (!Misc::PathCreate(destFilename))
		return false;

	// See if the destination file exists.
	DiskFile destFile;
	if (!destFile.Open(destFilename, File::MODE_CREATE | File::MODE_WRITEONLY))
		return false;

	// Allocate the buffer space.
	BYTE* buffer = (BYTE*)alloca(BUFFER_SIZE);

	// Keep copying until there is no more file left to copy.
	while (fileSize > 0) {
		// Copy the minimum of BUFFER_SIZE or the fileSize.
		uint32_t readSize = BUFFER_SIZE < fileSize ? BUFFER_SIZE : fileSize;
		if (srcFile.Read(buffer, readSize) != readSize)
			return false;
		destFile.Write(buffer, readSize);
		fileSize -= readSize;
	}

	// Close the destination virtual file.
	destFile.Close();

	return true;
}
예제 #8
0
// Scan any extra files specified on the command line
bool Par1Repairer::VerifyExtraFiles(const list<CommandLine::ExtraFile> &extrafiles) {
	for (ExtraFileIterator i=extrafiles.begin();
			 i!=extrafiles.end() && completefilecount<sourcefiles.size();
			 ++i) {
		string filename = i->FileName();

		bool skip = false;

		// Find the file extension
		string::size_type where = filename.find_last_of('.');
		if (where != string::npos) {
			string tail = filename.substr(where+1);

			// Check the the file extension is the correct form
			if ((tail[0] == 'P' || tail[0] == 'p') &&
					(
						((tail[1] == 'A' || tail[1] == 'a') && (tail[2] == 'R' || tail[2] == 'r'))
						||
						(isdigit(tail[1]) && isdigit(tail[2]))
					)) {
				skip = true;
			}
		}

		if (!skip) {
			filename = DiskFile::GetCanonicalPathname(filename);

			// Has this file already been dealt with
			if (diskfilemap.Find(filename) == 0) {
				DiskFile *diskfile = new DiskFile;

				// Does the file exist
				if (!diskfile->Open(filename)) {
					delete diskfile;
					continue;
				}

				// Remember that we have processed this file
				bool success = diskfilemap.Insert(diskfile);
				assert(success);

				// Do the actual verification
				VerifyDataFile(diskfile, 0);
				// Ignore errors

				// We have finished with the file for now
				diskfile->Close();

				// Find out how much data we have found
				UpdateVerificationResults();
			}
		}
	}

	return true;
}
예제 #9
0
// Verify that all of the reconstructed target files are now correct
bool Par1Repairer::VerifyTargetFiles(void) {
	bool finalresult = true;

	// Verify the target files in alphabetical order
//  sort(verifylist.begin(), verifylist.end(), SortSourceFilesByFileName);

	// Iterate through each file in the verification list
	for (list<Par1RepairerSourceFile*>::iterator sf = verifylist.begin();
			 sf != verifylist.end();
			 ++sf) {
		Par1RepairerSourceFile *sourcefile = *sf;
		DiskFile *targetfile = sourcefile->GetTargetFile();

		// Close the file
		if (targetfile->IsOpen())
			targetfile->Close();

		// Say we don't have a complete version of the file
		sourcefile->SetCompleteFile(0);

		// Re-open the target file
		if (!targetfile->Open()) {
			finalresult = false;
			continue;
		}

		// Verify the file again
		if (!VerifyDataFile(targetfile, sourcefile))
			finalresult = false;

		// Close the file again
		targetfile->Close();

		// Find out how much data we have found
		UpdateVerificationResults();
	}

	return finalresult;
}
예제 #10
0
bool FileSystem::SaveBufferIntoFile(const char* filename, const char* buffer, int bufLen)
{
	DiskFile file;
	if (!file.Open(filename, DiskFile::omWrite))
	{
		return false;
	}

	int writtenBytes = (int)file.Write(buffer, bufLen);
	file.Close();

	return writtenBytes == bufLen;
}
예제 #11
0
void ParRenamer::CheckRegularFile(const char* destDir, const char* filename)
{
	debug("Computing hash for %s", filename);

	DiskFile file;
	if (!file.Open(filename, DiskFile::omRead))
	{
		PrintMessage(Message::mkError, "Could not open file %s", filename);
		return;
	}

	// load first 16K of the file into buffer
	static const int blockSize = 16*1024;
	CharBuffer buffer(blockSize);

	int readBytes = (int)file.Read(buffer, buffer.Size());
	if (readBytes != buffer.Size() && file.Error())
	{
		PrintMessage(Message::mkError, "Could not read file %s", filename);
		return;
	}

	file.Close();

	Par2::MD5Hash hash16k;
	Par2::MD5Context context;
	context.Update(buffer, readBytes);
	context.Final(hash16k);

	debug("file: %s; hash16k: %s", FileSystem::BaseFileName(filename), hash16k.print().c_str());

	for (FileHash& fileHash : m_fileHashList)
	{
		if (!strcmp(fileHash.GetHash(), hash16k.print().c_str()))
		{
			debug("Found correct filename: %s", fileHash.GetFilename());
			fileHash.SetFileExists(true);

			BString<1024> dstFilename("%s%c%s", destDir, PATH_SEPARATOR, fileHash.GetFilename());

			if (!FileSystem::FileExists(dstFilename) && !IsSplittedFragment(filename, fileHash.GetFilename()))
			{
				RenameFile(filename, dstFilename);
			}

			break;
		}
	}
}
예제 #12
0
파일: Log.cpp 프로젝트: outtahere/nzbget
void Log::Filelog(const char* msg, ...)
{
	if (m_logFilename.Empty())
	{
		return;
	}

	char tmp2[1024];

	va_list ap;
	va_start(ap, msg);
	vsnprintf(tmp2, 1024, msg, ap);
	tmp2[1024-1] = '\0';
	va_end(ap);

	time_t rawtime = Util::CurrentTime() + g_Options->GetTimeCorrection();

	char time[50];
	Util::FormatTime(rawtime, time, 50);

	if ((int)rawtime/86400 != (int)m_lastWritten/86400 && g_Options->GetWriteLog() == Options::wlRotate)
	{
		RotateLog();
	}

	m_lastWritten = rawtime;

	DiskFile file;
	if (file.Open(m_logFilename, DiskFile::omAppend))
	{
#ifdef WIN32
		uint64 processId = GetCurrentProcessId();
		uint64 threadId = GetCurrentThreadId();
#else
		uint64 processId = (uint64)getpid();
		uint64 threadId = (uint64)pthread_self();
#endif
#ifdef DEBUG
		file.Print("%s\t%llu\t%llu\t%s%s", time, processId, threadId, tmp2, LINE_ENDING);
#else
		file.Print("%s\t%s%s", time, tmp2, LINE_ENDING);
#endif
		file.Close();
	}
	else
	{
		perror(m_logFilename);
	}
}
예제 #13
0
void ParRenamer::CheckParFile(const char* destDir, const char* filename)
{
	debug("Checking par2-header for %s", filename);

	DiskFile file;
	if (!file.Open(filename, DiskFile::omRead))
	{
		PrintMessage(Message::mkError, "Could not open file %s", filename);
		return;
	}

	// load par2-header
	Par2::PACKET_HEADER header;

	int readBytes = (int)file.Read(&header, sizeof(header));
	if (readBytes != sizeof(header) && file.Error())
	{
		PrintMessage(Message::mkError, "Could not read file %s", filename);
		return;
	}

	file.Close();

	// Check the packet header
	if (Par2::packet_magic != header.magic ||          // not par2-file
		sizeof(Par2::PACKET_HEADER) > header.length || // packet length is too small
		0 != (header.length & 3) ||              // packet length is not a multiple of 4
		FileSystem::FileSize(filename) < (int)header.length)       // packet would extend beyond the end of the file
	{
		// not par2-file or damaged header, ignoring the file
		return;
	}

	BString<100> setId = header.setid.print().c_str();
	for (char* p = setId; *p; p++) *p = tolower(*p); // convert string to lowercase

	debug("Storing: %s; setid: %s", FileSystem::BaseFileName(filename), *setId);

	m_parInfoList.emplace_back(filename, setId);
}
예제 #14
0
void WinConsole::SetupConfigFile()
{
	// create new config-file from config template

	char commonAppDataPath[MAX_PATH];
	SHGetFolderPath(nullptr, CSIDL_COMMON_APPDATA, nullptr, 0, commonAppDataPath);

	BString<1024> filename("%s\\NZBGet\\nzbget.conf", commonAppDataPath);

	BString<1024> appDataPath("%s\\NZBGet", commonAppDataPath);
	FileSystem::CreateDirectory(appDataPath);

	BString<1024> confTemplateFilename("%s\\nzbget.conf.template", g_Options->GetAppDir());
	CopyFile(confTemplateFilename, filename, FALSE);

	// set MainDir in the config-file
	int size = 0;
	CharBuffer config;
	if (FileSystem::LoadFileIntoBuffer(filename, config, true))
	{
		const char* SIGNATURE = "MainDir=${AppDir}\\downloads";
		char* p = strstr(config, SIGNATURE);
		if (p)
		{
			DiskFile outfile;
			if (outfile.Open(filename, DiskFile::omWrite))
			{
				outfile.Write(config, p - config);
				outfile.Write("MainDir=", 8);
				outfile.Write(appDataPath, strlen(appDataPath));
				outfile.Write(p + strlen(SIGNATURE), config.Size() - 1 - (p + strlen(SIGNATURE) - config) - 1);
				outfile.Close();
			}
		}
	}

	// create default destination directory (which is not created on start automatically)
	BString<1024> completeDir("%s\\NZBGet\\complete", commonAppDataPath);
	FileSystem::CreateDirectory(completeDir);
}
예제 #15
0
void ArticleWriter::CompleteFileParts()
{
	debug("Completing file parts");
	debug("ArticleFilename: %s", m_fileInfo->GetFilename());

	bool directWrite = (g_Options->GetDirectWrite() || m_fileInfo->GetForceDirectWrite()) && m_fileInfo->GetOutputInitialized();

	BString<1024> nzbName;
	BString<1024> nzbDestDir;
	BString<1024> filename;

	{
		GuardedDownloadQueue guard = DownloadQueue::Guard();
		nzbName = m_fileInfo->GetNzbInfo()->GetName();
		nzbDestDir = m_fileInfo->GetNzbInfo()->GetDestDir();
		filename = m_fileInfo->GetFilename();
	}

	BString<1024> infoFilename("%s%c%s", *nzbName, PATH_SEPARATOR, *filename);

	bool cached = m_fileInfo->GetCachedArticles() > 0;

	if (g_Options->GetRawArticle())
	{
		detail("Moving articles for %s", *infoFilename);
	}
	else if (directWrite && cached)
	{
		detail("Writing articles for %s", *infoFilename);
	}
	else if (directWrite)
	{
		detail("Checking articles for %s", *infoFilename);
	}
	else
	{
		detail("Joining articles for %s", *infoFilename);
	}

	// Ensure the DstDir is created
	CString errmsg;
	if (!FileSystem::ForceDirectories(nzbDestDir, errmsg))
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
			"Could not create directory %s: %s", *nzbDestDir, *errmsg);
		return;
	}

	CString ofn;
	if (m_fileInfo->GetForceDirectWrite())
	{
		ofn.Format("%s%c%s", *nzbDestDir, PATH_SEPARATOR, *filename);
	}
	else
	{
		ofn = FileSystem::MakeUniqueFilename(nzbDestDir, *filename);
	}

	DiskFile outfile;
	BString<1024> tmpdestfile("%s.tmp", *ofn);

	if (!g_Options->GetRawArticle() && !directWrite)
	{
		FileSystem::DeleteFile(tmpdestfile);
		if (!outfile.Open(tmpdestfile, DiskFile::omWrite))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not create file %s: %s", *tmpdestfile, *FileSystem::GetLastErrorMessage());
			return;
		}
	}
	else if (directWrite && cached)
	{
		if (!outfile.Open(m_outputFilename, DiskFile::omReadWrite))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not open file %s: %s", *m_outputFilename, *FileSystem::GetLastErrorMessage());
			return;
		}
		tmpdestfile = *m_outputFilename;
	}
	else if (g_Options->GetRawArticle())
	{
		FileSystem::DeleteFile(tmpdestfile);
		if (!FileSystem::CreateDirectory(ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not create directory %s: %s", *ofn, *FileSystem::GetLastErrorMessage());
			return;
		}
	}

	if (outfile.Active())
	{
		SetWriteBuffer(outfile, 0);
	}

	uint32 crc = 0;

	{
		std::unique_ptr<ArticleCache::FlushGuard> flushGuard;
		if (cached)
		{
			flushGuard = std::make_unique<ArticleCache::FlushGuard>(g_ArticleCache->GuardFlush());
		}

		CharBuffer buffer;
		bool firstArticle = true;

		if (!g_Options->GetRawArticle() && !directWrite)
		{
			buffer.Reserve(1024 * 64);
		}

		for (ArticleInfo* pa : m_fileInfo->GetArticles())
		{
			if (pa->GetStatus() != ArticleInfo::aiFinished)
			{
				continue;
			}

			if (!g_Options->GetRawArticle() && !directWrite && pa->GetSegmentOffset() > -1 &&
				pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1)
			{
				memset(buffer, 0, buffer.Size());
				if (!g_Options->GetSkipWrite())
				{
					while (pa->GetSegmentOffset() > outfile.Position() && outfile.Position() > -1 &&
						outfile.Write(buffer, std::min((int)(pa->GetSegmentOffset() - outfile.Position()), buffer.Size())));
				}
			}

			if (pa->GetSegmentContent())
			{
				if (!g_Options->GetSkipWrite())
				{
					outfile.Seek(pa->GetSegmentOffset());
					outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
				}
				pa->DiscardSegment();
			}
			else if (!g_Options->GetRawArticle() && !directWrite && !g_Options->GetSkipWrite())
			{
				DiskFile infile;
				if (pa->GetResultFilename() && infile.Open(pa->GetResultFilename(), DiskFile::omRead))
				{
					int cnt = buffer.Size();
					while (cnt == buffer.Size())
					{
						cnt = (int)infile.Read(buffer, buffer.Size());
						outfile.Write(buffer, cnt);
					}
					infile.Close();
				}
				else
				{
					m_fileInfo->SetFailedArticles(m_fileInfo->GetFailedArticles() + 1);
					m_fileInfo->SetSuccessArticles(m_fileInfo->GetSuccessArticles() - 1);
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not find file %s for %s [%i/%i]",
						pa->GetResultFilename(), *infoFilename, pa->GetPartNumber(),
						(int)m_fileInfo->GetArticles()->size());
				}
			}
			else if (g_Options->GetRawArticle())
			{
				BString<1024> dstFileName("%s%c%03i", *ofn, PATH_SEPARATOR, pa->GetPartNumber());
				if (!FileSystem::MoveFile(pa->GetResultFilename(), dstFileName))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not move file %s to %s: %s", pa->GetResultFilename(),
						*dstFileName, *FileSystem::GetLastErrorMessage());
				}
			}

			if (m_format == Decoder::efYenc)
			{
				crc = firstArticle ? pa->GetCrc() : Crc32::Combine(crc, pa->GetCrc(), pa->GetSegmentSize());
				firstArticle = false;
			}
		}

		buffer.Clear();
	}

	if (outfile.Active())
	{
		outfile.Close();
		if (!directWrite && !FileSystem::MoveFile(tmpdestfile, ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not move file %s to %s: %s", *tmpdestfile, *ofn,
				*FileSystem::GetLastErrorMessage());
		}
	}

	if (directWrite)
	{
		if (!FileSystem::SameFilename(m_outputFilename, ofn) &&
			!FileSystem::MoveFile(m_outputFilename, ofn))
		{
			m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
				"Could not move file %s to %s: %s", *m_outputFilename, *ofn,
				*FileSystem::GetLastErrorMessage());
		}

		// if destination directory was changed delete the old directory (if empty)
		int len = strlen(nzbDestDir);
		if (!(!strncmp(nzbDestDir, m_outputFilename, len) &&
			(m_outputFilename[len] == PATH_SEPARATOR || m_outputFilename[len] == ALT_PATH_SEPARATOR)))
		{
			debug("Checking old dir for: %s", *m_outputFilename);
			BString<1024> oldDestDir;
			oldDestDir.Set(m_outputFilename, (int)(FileSystem::BaseFileName(m_outputFilename) - m_outputFilename));
			if (FileSystem::DirEmpty(oldDestDir))
			{
				debug("Deleting old dir: %s", *oldDestDir);
				FileSystem::RemoveDirectory(oldDestDir);
			}
		}
	}

	if (!directWrite)
	{
		for (ArticleInfo* pa : m_fileInfo->GetArticles())
		{
			if (pa->GetResultFilename())
			{
				FileSystem::DeleteFile(pa->GetResultFilename());
			}
		}
	}

	if (m_fileInfo->GetTotalArticles() == m_fileInfo->GetSuccessArticles())
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Successfully downloaded %s", *infoFilename);
	}
	else if (m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles() > 0)
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkWarning,
			"%i of %i article downloads failed for \"%s\"",
			m_fileInfo->GetMissedArticles() + m_fileInfo->GetFailedArticles(),
			m_fileInfo->GetTotalArticles(), *infoFilename);
	}
	else
	{
		m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkInfo, "Partially downloaded %s", *infoFilename);
	}

	{
		GuardedDownloadQueue guard = DownloadQueue::Guard();

		m_fileInfo->SetCrc(crc);
		m_fileInfo->SetOutputFilename(ofn);

		if (strcmp(m_fileInfo->GetFilename(), filename))
		{
			// file was renamed during completion, need to move the file
			ofn = FileSystem::MakeUniqueFilename(nzbDestDir, m_fileInfo->GetFilename());
			if (!FileSystem::MoveFile(m_fileInfo->GetOutputFilename(), ofn))
			{
				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
					"Could not rename file %s to %s: %s", m_fileInfo->GetOutputFilename(),
					*ofn, *FileSystem::GetLastErrorMessage());
			}
			m_fileInfo->SetOutputFilename(ofn);
		}

		if (strcmp(m_fileInfo->GetNzbInfo()->GetDestDir(), nzbDestDir))
		{
			// destination directory was changed during completion, need to move the file
			MoveCompletedFiles(m_fileInfo->GetNzbInfo(), nzbDestDir);
		}
	}
}
예제 #16
0
// Read source data, process it through the RS matrix and write it to disk.
bool Par2Creator::ProcessData(u64 blockoffset, size_t blocklength)
{
  // Clear the output buffer
  memset(outputbuffer, 0, chunksize * recoveryblockcount);

  // If we have defered computation of the file hash and block crc and hashes
  // sourcefile and sourceindex will be used to update them during
  // the main recovery block computation
  vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
  u32 sourceindex = 0;

  vector<DataBlock>::iterator sourceblock;
  u32 inputblock;

  DiskFile *lastopenfile = NULL;

  // For each input block
  for ((sourceblock=sourceblocks.begin()),(inputblock=0);
       sourceblock != sourceblocks.end();
       ++sourceblock, ++inputblock)
  {
    // Are we reading from a new file?
    if (lastopenfile != (*sourceblock).GetDiskFile())
    {
      // Close the last file
      if (lastopenfile != NULL)
      {
        lastopenfile->Close();
      }

      // Open the new file
      lastopenfile = (*sourceblock).GetDiskFile();
      if (!lastopenfile->Open())
      {
        return false;
      }
    }

    // Read data from the current input block
    if (!sourceblock->ReadData(blockoffset, blocklength, inputbuffer))
      return false;

    if (deferhashcomputation)
    {
      assert(blockoffset == 0 && blocklength == blocksize);
      assert(sourcefile != sourcefiles.end());

      (*sourcefile)->UpdateHashes(sourceindex, inputbuffer, blocklength);
    }

	// Function that does the subtask in multiple threads if appropriate.
	if (!this->CreateParityBlocks (blocklength, inputblock))
		return false;
	
    // Work out which source file the next block belongs to
    if (++sourceindex >= (*sourcefile)->BlockCount())
    {
      sourceindex = 0;
      ++sourcefile;
    }
  }

  // Close the last file
  if (lastopenfile != NULL)
  {
    lastopenfile->Close();
  }

  if (noiselevel > CommandLine::nlQuiet)
    cout << "Writing recovery packets\r";

  // For each output block
  for (u32 outputblock=0; outputblock<recoveryblockcount;outputblock++)
  {
    // Select the appropriate part of the output buffer
    char *outbuf = &((char*)outputbuffer)[chunksize * outputblock];

    // Write the data to the recovery packet
    if (!recoverypackets[outputblock].WriteData(blockoffset, blocklength, outbuf))
      return false;
  }

  if (noiselevel > CommandLine::nlQuiet)
    cout << "Wrote " << recoveryblockcount * blocklength << " bytes to disk" << endl;

  return true;
}
예제 #17
0
// Attempt to verify all of the source files
bool Par1Repairer::VerifySourceFiles(void) {
	bool finalresult = true;

	u32 filenumber = 0;
	vector<Par1RepairerSourceFile*>::iterator sourceiterator = sourcefiles.begin();
	while (sourceiterator != sourcefiles.end()) {
		Par1RepairerSourceFile *sourcefile = *sourceiterator;

		string filename = sourcefile->FileName();

		// Check to see if we have already used this file
		if (diskfilemap.Find(filename) != 0) {
			string path;
			string name;
			DiskFile::SplitRelativeFilename(filename, path, name);

			// The file has already been used!
//			cerr << "Source file " << filenumber+1 << " is a duplicate." << endl; // Original
			cerr << "Source file " << name         << " is a duplicate." << endl; // Imported
			finalresult = false;
			} else {

			DiskFile *diskfile = new DiskFile;

			// Does the target file exist
			if (diskfile->Open(filename)) {
				// Yes. Record that fact.
				sourcefile->SetTargetExists(true);

				// Remember that the DiskFile is the target file
				sourcefile->SetTargetFile(diskfile);

				// Remember that we have processed this file
				bool success = diskfilemap.Insert(diskfile);
				assert(success);

				// Do the actual verification
				if (!VerifyDataFile(diskfile, sourcefile))
					finalresult = false;

				// We have finished with the file for now
				diskfile->Close();

				// Find out how much data we have found
				UpdateVerificationResults();
			} else {
				// The file does not exist.
				delete diskfile;

				if (noiselevel > CommandLine::nlSilent) {
					string path;
					string name;
					DiskFile::SplitFilename(filename, path, name);

					cout << "Target: \"" << name << "\" - missing." << endl;
				}
			}
		}
		++sourceiterator;
		++filenumber;
	}

	return finalresult;
}
예제 #18
0
bool Par1Repairer::LoadRecoveryFile(string filename) {
	// Skip the file if it has already been processed
	if (diskfilemap.Find(filename) != 0) {
		return true;
	}

	DiskFile *diskfile = new DiskFile;

	// Open the file
	if (!diskfile->Open(filename)) {
		// If we could not open the file, ignore the error and
		// proceed to the next file
		delete diskfile;
		return true;
	}

	if (noiselevel > CommandLine::nlSilent) {
		string path;
		string name;
		DiskFile::SplitFilename(filename, path, name);
		cout << "Loading \"" << name << "\"." << endl;
	}

	parlist.push_back(filename);

	bool havevolume = false;
	u32 volumenumber = 0;

	// How big is the file
	u64 filesize = diskfile->FileSize();
	if (filesize >= sizeof(PAR1FILEHEADER)) {
		// Allocate a buffer to read data into
		size_t buffersize = (size_t)min((u64)1048576, filesize);
		u8 *buffer = new u8[buffersize];

		do {
			PAR1FILEHEADER fileheader;
			if (!diskfile->Read(0, &fileheader, sizeof(fileheader)))
				break;

			// Is this really a PAR file?
			if (fileheader.magic != par1_magic)
				break;

			// Is the version number correct?
			if (fileheader.fileversion != 0x00010000)
				break;

			ignore16kfilehash = (fileheader.programversion == smartpar11);

			// Prepare to carry out MD5 Hash check of the Control Hash
			MD5Context context;
			u64 offset = offsetof(PAR1FILEHEADER, sethash);

			// Process until the end of the file is reached
			while (offset < filesize) {
				// How much data should we read?
				size_t want = (size_t)min((u64)buffersize, filesize-offset);
				if (!diskfile->Read(offset, buffer, want))
					break;

				context.Update(buffer, want);

				offset += want;
			}

			// Did we read the whole file
			if (offset < filesize)
				break;

			// Compute the hash value
			MD5Hash hash;
			context.Final(hash);

			// Is it correct?
			if (hash != fileheader.controlhash)
				break;

			// Check that the volume number is ok
			if (fileheader.volumenumber >= 256)
				break;

			// Are there any files?
			if (fileheader.numberoffiles == 0 ||
					fileheader.filelistoffset < sizeof(PAR1FILEHEADER) ||
					fileheader.filelistsize == 0)
				break;

			// Verify that the file list and data offsets are ok
			if ((fileheader.filelistoffset + fileheader.filelistsize > filesize)
					||
					(fileheader.datasize && (fileheader.dataoffset < sizeof(fileheader) || fileheader.dataoffset + fileheader.datasize > filesize))
					||
					(fileheader.datasize && ((fileheader.filelistoffset <= fileheader.dataoffset && fileheader.dataoffset < fileheader.filelistoffset+fileheader.filelistsize) || (fileheader.dataoffset <= fileheader.filelistoffset && fileheader.filelistoffset < fileheader.dataoffset + fileheader.datasize))))
				break;

			// Check the size of the file list
			if (fileheader.filelistsize > 200000)
				break;

			// If we already have a copy of the file list, make sure this one has the same size
			if (filelist != 0 && filelistsize != fileheader.filelistsize)
				break;

			// Allocate a buffer to hold a copy of the file list
			unsigned char *temp = new unsigned char[(size_t)fileheader.filelistsize];

			// Read the file list into the buffer
			if (!diskfile->Read(fileheader.filelistoffset, temp, (size_t)fileheader.filelistsize)) {
				delete [] temp;
				break;
			}

			// If we already have a copy of the file list, make sure this copy is identical
			if (filelist != 0) {
				bool match = (0 == memcmp(filelist, temp, filelistsize));
				delete [] temp;

				if (!match)
					break;
			} else {
				// Prepare to scan the file list
				unsigned char *current = temp;
				size_t remaining = (size_t)fileheader.filelistsize;
				unsigned int fileindex = 0;

				// Allocate a buffer to copy each file entry into so that
				// all fields will be correctly aligned in memory.
				PAR1FILEENTRY *fileentry = (PAR1FILEENTRY*)new u64[(remaining + sizeof(u64)-1)/sizeof(u64)];

				// Process until we run out of files or data
				while (remaining > 0 && fileindex < fileheader.numberoffiles) {
					// Copy fixed portion of file entry
					memcpy((void*)fileentry, (void*)current, sizeof(PAR1FILEENTRY));

					// Is there enough data remaining
					if (remaining < sizeof(fileentry->entrysize) ||
							remaining < fileentry->entrysize)
						break;

					// Check the length of the filename
					if (fileentry->entrysize <= sizeof(PAR1FILEENTRY))
						break;

					// Check the file size
					if (blocksize < fileentry->filesize)
						blocksize = fileentry->filesize;

					// Copy whole of file entry
					memcpy((void*)fileentry, (void*)current, (size_t)(u64)fileentry->entrysize);

					// Create source file and add it to the appropriate list
					Par1RepairerSourceFile *sourcefile = new Par1RepairerSourceFile(fileentry, searchpath);
					if (fileentry->status & INPARITYVOLUME) {
						sourcefiles.push_back(sourcefile);
					} else {
						extrafiles.push_back(sourcefile);
					}

					remaining -= (size_t)fileentry->entrysize;
					current += (size_t)fileentry->entrysize;

					fileindex++;
				}

				delete [] (u64*)fileentry;

				// Did we find the correct number of files
				if (fileindex < fileheader.numberoffiles) {
					vector<Par1RepairerSourceFile*>::iterator i = sourcefiles.begin();
					while (i != sourcefiles.end()) {
						Par1RepairerSourceFile *sourcefile = *i;
						delete sourcefile;
						++i;
					}
					sourcefiles.clear();

					i = extrafiles.begin();
					while (i != extrafiles.end()) {
						Par1RepairerSourceFile *sourcefile = *i;
						delete sourcefile;
						++i;
					}
					extrafiles.clear();

					delete [] temp;
					break;
				}

				filelist = temp;
				filelistsize = (u32)fileheader.filelistsize;
			}

			// Is this a recovery volume?
			if (fileheader.volumenumber > 0) {
				// Make sure there is data and that it is the correct size
				if (fileheader.dataoffset == 0 || fileheader.datasize != blocksize)
					break;

				// What volume number is this?
				volumenumber = (u32)(fileheader.volumenumber - 1);

				// Do we already have this volume?
				if (recoveryblocks.find(volumenumber) == recoveryblocks.end()) {
					// Create a data block
					DataBlock *datablock = new DataBlock;
					datablock->SetLength(blocksize);
					datablock->SetLocation(diskfile, fileheader.dataoffset);

					// Store it in the map
					recoveryblocks.insert(pair<u32, DataBlock*>(volumenumber, datablock));

					havevolume = true;
				}
			}
		} while (false);

		delete [] buffer;
	}

	// We have finished with the file for now
	diskfile->Close();

	if (noiselevel > CommandLine::nlQuiet) {
		if (havevolume) {
			cout << "Loaded recovery volume " << volumenumber << endl;
		} else {
			cout << "No new recovery volumes found" << endl;
		}
	}

	// Remember that the file was processed
	bool success = diskfilemap.Insert(diskfile);
	assert(success);

	return true;
}
예제 #19
0
void ArticleWriter::FlushCache()
{
	detail("Flushing cache for %s", *m_infoName);

	bool directWrite = g_Options->GetDirectWrite() && m_fileInfo->GetOutputInitialized();
	DiskFile outfile;
	bool needBufFile = false;
	int flushedArticles = 0;
	int64 flushedSize = 0;

	{
		ArticleCache::FlushGuard flushGuard = g_ArticleCache->GuardFlush();

		std::vector<ArticleInfo*> cachedArticles;

		{
			Guard contentGuard = g_ArticleCache->GuardContent();

			if (m_fileInfo->GetFlushLocked())
			{
				return;
			}

			m_fileInfo->SetFlushLocked(true);

			cachedArticles.reserve(m_fileInfo->GetArticles()->size());
			for (ArticleInfo* pa : m_fileInfo->GetArticles())
			{
				if (pa->GetSegmentContent())
				{
					cachedArticles.push_back(pa);
				}
			}
		}

		for (ArticleInfo* pa : cachedArticles)
		{
			if (m_fileInfo->GetDeleted() && !m_fileInfo->GetNzbInfo()->GetParking())
			{
				// the file was deleted during flushing: stop flushing immediately
				break;
			}

			if (directWrite && !outfile.Active())
			{
				if (!outfile.Open(m_fileInfo->GetOutputFilename(), DiskFile::omReadWrite))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not open file %s: %s", m_fileInfo->GetOutputFilename(),
						*FileSystem::GetLastErrorMessage());
					// prevent multiple error messages
					pa->DiscardSegment();
					flushedArticles++;
					break;
				}
				needBufFile = true;
			}

			BString<1024> destFile;

			if (!directWrite)
			{
				destFile.Format("%s.tmp", pa->GetResultFilename());
				if (!outfile.Open(destFile, DiskFile::omWrite))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not create file %s: %s", *destFile,
						*FileSystem::GetLastErrorMessage());
					// prevent multiple error messages
					pa->DiscardSegment();
					flushedArticles++;
					break;
				}
				needBufFile = true;
			}

			if (outfile.Active() && needBufFile)
			{
				SetWriteBuffer(outfile, 0);
				needBufFile = false;
			}

			if (directWrite)
			{
				outfile.Seek(pa->GetSegmentOffset());
			}

			if (!g_Options->GetSkipWrite())
			{
				outfile.Write(pa->GetSegmentContent(), pa->GetSegmentSize());
			}

			flushedSize += pa->GetSegmentSize();
			flushedArticles++;

			pa->DiscardSegment();

			if (!directWrite)
			{
				outfile.Close();

				if (!FileSystem::MoveFile(destFile, pa->GetResultFilename()))
				{
					m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError,
						"Could not rename file %s to %s: %s", *destFile, pa->GetResultFilename(),
						*FileSystem::GetLastErrorMessage());
				}
			}
		}

		outfile.Close();

		{
			Guard contentGuard = g_ArticleCache->GuardContent();
			m_fileInfo->SetCachedArticles(m_fileInfo->GetCachedArticles() - flushedArticles);
			m_fileInfo->SetFlushLocked(false);
		}
	}

	detail("Saved %i articles (%.2f MB) from cache into disk for %s", flushedArticles,
		(float)(flushedSize / 1024.0 / 1024.0), *m_infoName);
}
예제 #20
0
// Read source data, process it through the RS matrix and write it to disk.
bool Par2Creator::ProcessData(u64 blockoffset, size_t blocklength)
{
  // Clear the output buffer
  memset(outputbuffer, 0, chunksize * recoveryblockcount);

  // If we have defered computation of the file hash and block crc and hashes
  // sourcefile and sourceindex will be used to update them during
  // the main recovery block computation
  vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
  u32 sourceindex = 0;

  vector<DataBlock>::iterator sourceblock;
  u32 inputblock;

  DiskFile *lastopenfile = NULL;

  // For each input block
  for ((sourceblock=sourceblocks.begin()),(inputblock=0);
       sourceblock != sourceblocks.end();
       ++sourceblock, ++inputblock)
  {
    // Are we reading from a new file?
    if (lastopenfile != (*sourceblock).GetDiskFile())
    {
      // Close the last file
      if (lastopenfile != NULL)
      {
        lastopenfile->Close();
      }

      // Open the new file
      lastopenfile = (*sourceblock).GetDiskFile();
      if (!lastopenfile->Open())
      {
        return false;
      }
    }

    // Read data from the current input block
    if (!sourceblock->ReadData(blockoffset, blocklength, inputbuffer))
      return false;

    if (deferhashcomputation)
    {
      assert(blockoffset == 0 && blocklength == blocksize);
      assert(sourcefile != sourcefiles.end());

      (*sourcefile)->UpdateHashes(sourceindex, inputbuffer, blocklength);
    }

    // For each output block
    #pragma omp parallel for
    for (u32 outputblock=0; outputblock<recoveryblockcount; outputblock++)
    {
      // Select the appropriate part of the output buffer
      void *outbuf = &((u8*)outputbuffer)[chunksize * outputblock];

      // Process the data through the RS matrix
      rs.Process(blocklength, inputblock, inputbuffer, outputblock, outbuf);

      if (noiselevel > CommandLine::nlQuiet)
      {
        // Update a progress indicator
        u32 oldfraction = (u32)(1000 * progress / totaldata);
        #pragma omp atomic
        progress += blocklength;
        u32 newfraction = (u32)(1000 * progress / totaldata);

        if (oldfraction != newfraction)
        {
          #pragma omp critical
          cout << "Processing: " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush;
        }
      }
    }

    // Work out which source file the next block belongs to
    if (++sourceindex >= (*sourcefile)->BlockCount())
    {
      sourceindex = 0;
      ++sourcefile;
    }
  }

  // Close the last file
  if (lastopenfile != NULL)
  {
    lastopenfile->Close();
  }

  if (noiselevel > CommandLine::nlQuiet)
    cout << "Writing recovery packets\r";

  // For each output block
  for (u32 outputblock=0; outputblock<recoveryblockcount;outputblock++)
  {
    // Select the appropriate part of the output buffer
    char *outbuf = &((char*)outputbuffer)[chunksize * outputblock];

    // Write the data to the recovery packet
    if (!recoverypackets[outputblock].WriteData(blockoffset, blocklength, outbuf))
      return false;
  }

  if (noiselevel > CommandLine::nlQuiet)
    cout << "Wrote " << recoveryblockcount * blocklength << " bytes to disk" << endl;

  return true;
}