static bool ReadDataString(CFSTR fileName, LPCSTR startID,
LPCSTR endID, AString &stringResult)
{
stringResult.Empty();
NIO::CInFile inFile;
if (!inFile.Open(fileName))
return false;
const int kBufferSize = (1 << 12);
Byte buffer[kBufferSize];
int signatureStartSize = MyStringLen(startID);
int signatureEndSize = MyStringLen(endID);
UInt32 numBytesPrev = 0;
bool writeMode = false;
UInt64 posTotal = 0;
for (;;)
{
if (posTotal > (1 << 20))
return (stringResult.IsEmpty());
UInt32 numReadBytes = kBufferSize - numBytesPrev;
UInt32 processedSize;
if (!inFile.Read(buffer + numBytesPrev, numReadBytes, processedSize))
return false;
if (processedSize == 0)
return true;
UInt32 numBytesInBuffer = numBytesPrev + processedSize;
UInt32 pos = 0;
for (;;)
{
if (writeMode)
{
if (pos > numBytesInBuffer - signatureEndSize)
break;
if (memcmp(buffer + pos, endID, signatureEndSize) == 0)
return true;
char b = buffer[pos];
if (b == 0)
return false;
stringResult += b;
pos++;
}
else
{
if (pos > numBytesInBuffer - signatureStartSize)
break;
if (memcmp(buffer + pos, startID, signatureStartSize) == 0)
{
writeMode = true;
pos += signatureStartSize;
}
else
pos++;
}
}
numBytesPrev = numBytesInBuffer - pos;
posTotal += pos;
memmove(buffer, buffer + pos, numBytesPrev);
}
}
bool CFSFolder::LoadComments()
{
_comments.Clear();
_commentsAreLoaded = true;
NIO::CInFile file;
if (!file.Open(_path + kDescriptionFileName))
return false;
UInt64 len;
if (!file.GetLength(len))
return false;
if (len >= (1 << 28))
return false;
AString s;
char *p = s.GetBuf((unsigned)(size_t)len);
UInt32 processedSize;
file.Read(p, (UInt32)len, processedSize);
s.ReleaseBuf_CalcLen((unsigned)(size_t)len);
if (processedSize != len)
return false;
file.Close();
UString unicodeString;
if (!ConvertUTF8ToUnicode(s, unicodeString))
return false;
return _comments.ReadFromString(unicodeString);
}
bool CFSFolder::LoadComments()
{
if (_commentsAreLoaded)
return true;
_comments.Clear();
_commentsAreLoaded = true;
NIO::CInFile file;
if (!file.Open(_path + kDescriptionFileName))
return false;
UInt64 length;
if (!file.GetLength(length))
return false;
if (length >= (1 << 28))
return false;
AString s;
char *p = s.GetBuffer((int)((size_t)length + 1));
UInt32 processedSize;
file.Read(p, (UInt32)length, processedSize);
p[length] = 0;
s.ReleaseBuffer();
if (processedSize != length)
return false;
file.Close();
UString unicodeString;
if (!ConvertUTF8ToUnicode(s, unicodeString))
return false;
return _comments.ReadFromString(unicodeString);
}
static HRESULT CopyFileSpec(CFSTR fromPath, CFSTR toPath, bool writeToDisk, UInt64 fileSize,
UInt32 bufferSize, UInt64 progressStart, IProgress *progress)
{
NIO::CInFile inFile;
if (!inFile.Open(fromPath))
return GetLastError();
if (fileSize == (UInt64)(Int64)-1)
{
if (!inFile.GetLength(fileSize))
::GetLastError();
}
NIO::COutFile outFile;
if (writeToDisk)
{
if (!outFile.Open(toPath, FILE_SHARE_WRITE, OPEN_EXISTING, 0))
return GetLastError();
}
else
if (!outFile.Create(toPath, true))
return GetLastError();
CPhysTempBuffer tempBuffer;
tempBuffer.buffer = MidAlloc(bufferSize);
if (!tempBuffer.buffer)
return E_OUTOFMEMORY;
for (UInt64 pos = 0; pos < fileSize;)
{
UInt64 progressCur = progressStart + pos;
RINOK(progress->SetCompleted(&progressCur));
UInt64 rem = fileSize - pos;
UInt32 curSize = (UInt32)MyMin(rem, (UInt64)bufferSize);
UInt32 processedSize;
if (!inFile.Read(tempBuffer.buffer, curSize, processedSize))
return GetLastError();
if (processedSize == 0)
break;
curSize = processedSize;
if (writeToDisk)
{
const UInt32 kMask = 0x1FF;
curSize = (curSize + kMask) & ~kMask;
if (curSize > bufferSize)
return E_FAIL;
}
if (!outFile.Write(tempBuffer.buffer, curSize, processedSize))
return GetLastError();
if (curSize != processedSize)
return E_FAIL;
pos += curSize;
}
return S_OK;
}
HRESULT CThreadCombine::ProcessVirt()
{
NIO::COutFile outFile;
if (!outFile.Create(OutputPath, false))
{
HRESULT res = GetLastError();
AddErrorPath(OutputPath);
return res;
}
CProgressSync &sync = Sync;
sync.Set_NumBytesTotal(TotalSize);
CMyBuffer bufferObject;
if (!bufferObject.Allocate(kBufSize))
return E_OUTOFMEMORY;
Byte *buffer = (Byte *)(void *)bufferObject;
UInt64 pos = 0;
FOR_VECTOR (i, Names)
{
NIO::CInFile inFile;
const FString nextName = InputDirPrefix + Names[i];
if (!inFile.Open(nextName))
{
HRESULT res = GetLastError();
AddErrorPath(nextName);
return res;
}
sync.Set_FilePath(fs2us(nextName));
for (;;)
{
UInt32 processedSize;
if (!inFile.Read(buffer, kBufSize, processedSize))
{
HRESULT res = GetLastError();
AddErrorPath(nextName);
return res;
}
if (processedSize == 0)
break;
UInt32 needSize = processedSize;
if (!outFile.Write(buffer, needSize, processedSize))
{
HRESULT res = GetLastError();
AddErrorPath(OutputPath);
return res;
}
if (needSize != processedSize)
throw g_Message_FileWriteError;
pos += processedSize;
RINOK(sync.Set_NumBytesCur(pos));
}
}
HRESULT CCopyStateIO::MyCopyFile(CFSTR inPath, CFSTR outPath, DWORD attrib)
{
ErrorFileIndex = -1;
ErrorMessage.Empty();
CurrentSize = 0;
{
const size_t kBufSize = 1 << 16;
CByteArr buf(kBufSize);
NIO::CInFile inFile;
NIO::COutFile outFile;
if (!inFile.Open(inPath))
{
ErrorFileIndex = 0;
return S_OK;
}
if (!outFile.Create(outPath, true))
{
ErrorFileIndex = 1;
return S_OK;
}
for (;;)
{
UInt32 num;
if (!inFile.Read(buf, kBufSize, num))
{
ErrorFileIndex = 0;
return S_OK;
}
if (num == 0)
break;
UInt32 written = 0;
if (!outFile.Write(buf, num, written))
{
ErrorFileIndex = 1;
return S_OK;
}
if (written != num)
{
ErrorMessage = "Write error";
return S_OK;
}
CurrentSize += num;
if (Progress)
{
UInt64 completed = StartPos + CurrentSize;
RINOK(Progress->SetCompleted(&completed));
}
}
}
if (attrib != INVALID_FILE_ATTRIBUTES)
SetFileAttrib(outPath, attrib);
if (DeleteSrcFile)
{
if (!DeleteFileAlways(inPath))
{
ErrorFileIndex = 0;
return S_OK;
}
}
return S_OK;
}
HRESULT CThreadCrc::ProcessVirt()
{
Hash.Init();
CMyBuffer buf;
if (!buf.Allocate(kBufSize))
return E_OUTOFMEMORY;
CProgressSync &sync = ProgressDialog.Sync;
SetStatus(LangString(IDS_SCANNING));
Enumerator.Init();
FString path;
NFind::CFileInfo fi;
UInt64 numFiles = 0;
UInt64 numItems = 0, numItems_Prev = 0;
UInt64 totalSize = 0;
for (;;)
{
bool filled;
DWORD error = Enumerator.GetNextFile(fi, filled, path);
if (error != 0)
{
AddErrorMessage(error, path);
continue;
}
if (!filled)
break;
if (!fi.IsDir())
{
totalSize += fi.Size;
numFiles++;
}
numItems++;
bool needPrint = false;
// if (fi.IsDir())
{
if (numItems - numItems_Prev >= 100)
{
needPrint = true;
numItems_Prev = numItems;
}
}
/*
else if (numFiles - numFiles_Prev >= 200)
{
needPrint = true;
numFiles_Prev = numFiles;
}
*/
if (needPrint)
{
RINOK(sync.ScanProgress(numFiles, totalSize, path, fi.IsDir()));
}
}
RINOK(sync.ScanProgress(numFiles, totalSize, FString(), false));
// sync.SetNumFilesTotal(numFiles);
// sync.SetProgress(totalSize, 0);
// SetStatus(LangString(IDS_CHECKSUM_CALCULATING));
// sync.SetCurFilePath(L"");
SetStatus(L"");
Enumerator.Init();
FString tempPath;
FString firstFilePath;
bool isFirstFile = true;
UInt64 errorsFilesSize = 0;
for (;;)
{
bool filled;
DWORD error = Enumerator.GetNextFile(fi, filled, path);
if (error != 0)
{
AddErrorMessage(error, path);
continue;
}
if (!filled)
break;
error = 0;
Hash.InitForNewFile();
if (!fi.IsDir())
{
NIO::CInFile inFile;
tempPath = Enumerator.BasePrefix_for_Open;
tempPath += path;
if (!inFile.Open(tempPath))
{
error = GetNormalizedError();
AddErrorMessage(error, path);
continue;
}
if (isFirstFile)
{
firstFilePath = path;
isFirstFile = false;
}
sync.Set_FilePath(fs2us(path));
sync.Set_NumFilesCur(Hash.NumFiles);
UInt64 progress_Prev = 0;
for (;;)
{
UInt32 size;
if (!inFile.Read(buf, kBufSize, size))
{
error = GetNormalizedError();
AddErrorMessage(error, path);
UInt64 errorSize = 0;
if (inFile.GetLength(errorSize))
errorsFilesSize += errorSize;
break;
}
if (size == 0)
break;
Hash.Update(buf, size);
if (Hash.CurSize - progress_Prev >= ((UInt32)1 << 21))
{
RINOK(sync.Set_NumBytesCur(errorsFilesSize + Hash.FilesSize + Hash.CurSize));
progress_Prev = Hash.CurSize;
}
}
}
if (error == 0)
Hash.Final(fi.IsDir(), false, fs2us(path));
RINOK(sync.Set_NumBytesCur(errorsFilesSize + Hash.FilesSize));
}
RINOK(sync.Set_NumBytesCur(errorsFilesSize + Hash.FilesSize));
sync.Set_NumFilesCur(Hash.NumFiles);
if (Hash.NumFiles != 1)
sync.Set_FilePath(L"");
SetStatus(L"");
CProgressMessageBoxPair &pair = GetMessagePair(Hash.NumErrors != 0);
AddHashBundleRes(pair.Message, Hash, fs2us(firstFilePath));
LangString(IDS_CHECKSUM_INFORMATION, pair.Title);
return S_OK;
}
HRESULT CThreadCrc::ProcessVirt()
{
DataSize = NumFolders = NumFiles = NumFilesScan = DataCrcSum = DataNameCrcSum = 0;
memset(Sha256Sum, 0, SHA256_DIGEST_SIZE);
// ProgressDialog.WaitCreating();
CMyBuffer bufferObject;
if (!bufferObject.Allocate(kBufSize))
return E_OUTOFMEMORY;
Byte *buffer = (Byte *)(void *)bufferObject;
UInt64 totalSize = 0;
Enumerator.Init();
UString scanningStr = LangString(IDS_SCANNING, 0x03020800);
scanningStr += L' ';
CProgressSync &sync = ProgressDialog.Sync;
for (;;)
{
NFind::CFileInfo fileInfo;
bool filled;
FString resPath;
HRESULT errorCode = Enumerator.GetNextFile(fileInfo, filled, resPath);
if (errorCode != 0)
{
SetErrorPath1(resPath);
return errorCode;
}
if (!filled)
break;
if (!fileInfo.IsDir())
{
totalSize += fileInfo.Size;
NumFilesScan++;
}
sync.SetCurrentFileName(scanningStr + fs2us(resPath));
sync.SetProgress(totalSize, 0);
RINOK(sync.SetPosAndCheckPaused(0));
}
sync.SetNumFilesTotal(NumFilesScan);
sync.SetProgress(totalSize, 0);
Enumerator.Init();
for (;;)
{
NFind::CFileInfo fileInfo;
bool filled;
FString resPath;
HRESULT errorCode = Enumerator.GetNextFile(fileInfo, filled, resPath);
if (errorCode != 0)
{
SetErrorPath1(resPath);
return errorCode;
}
if (!filled)
break;
UInt32 crc = CRC_INIT_VAL;
CSha256 sha256;
Sha256_Init(&sha256);
if (fileInfo.IsDir())
NumFolders++;
else
{
NIO::CInFile inFile;
if (!inFile.Open(Enumerator.BasePrefix + resPath))
{
errorCode = GetNormalizedError();
SetErrorPath1(resPath);
return errorCode;
}
sync.SetCurrentFileName(fs2us(resPath));
sync.SetNumFilesCur(NumFiles);
NumFiles++;
for (;;)
{
UInt32 processedSize;
if (!inFile.Read(buffer, kBufSize, processedSize))
{
errorCode = GetNormalizedError();
SetErrorPath1(resPath);
return errorCode;
}
if (processedSize == 0)
break;
crc = CrcUpdate(crc, buffer, processedSize);
if (NumFilesScan == 1)
Sha256_Update(&sha256, buffer, processedSize);
DataSize += processedSize;
RINOK(sync.SetPosAndCheckPaused(DataSize));
}
DataCrcSum += CRC_GET_DIGEST(crc);
if (NumFilesScan == 1)
Sha256_Final(&sha256, Sha256Sum);
}
for (int i = 0; i < resPath.Length(); i++)
{
wchar_t c = resPath[i];
crc = CRC_UPDATE_BYTE(crc, ((Byte)(c & 0xFF)));
crc = CRC_UPDATE_BYTE(crc, ((Byte)((c >> 8) & 0xFF)));
}
DataNameCrcSum += CRC_GET_DIGEST(crc);
RINOK(sync.SetPosAndCheckPaused(DataSize));
}
sync.SetNumFilesCur(NumFiles);
OkMessage = GetResultMessage();
OkMessageTitle = LangString(IDS_CHECKSUM_INFORMATION, 0x03020720);
return S_OK;
}
HRESULT CThreadSplit::ProcessVirt()
{
NIO::CInFile inFile;
if (!inFile.Open(FilePath))
return GetLastError();
CPreAllocOutFile outFile;
CMyBuffer buffer;
if (!buffer.Allocate(kBufSize))
return E_OUTOFMEMORY;
CVolSeqName seqName;
seqName.SetNumDigits(NumVolumes);
UInt64 length;
if (!inFile.GetLength(length))
return GetLastError();
CProgressSync &sync = Sync;
sync.Set_NumBytesTotal(length);
UInt64 pos = 0;
UInt64 prev = 0;
UInt64 numFiles = 0;
unsigned volIndex = 0;
for (;;)
{
UInt64 volSize;
if (volIndex < VolumeSizes.Size())
volSize = VolumeSizes[volIndex];
else
volSize = VolumeSizes.Back();
UInt32 needSize = kBufSize;
{
const UInt64 rem = volSize - outFile.Written;
if (needSize > rem)
needSize = (UInt32)rem;
}
UInt32 processedSize;
if (!inFile.Read(buffer, needSize, processedSize))
return GetLastError();
if (processedSize == 0)
return S_OK;
needSize = processedSize;
if (outFile.Written == 0)
{
FString name = VolBasePath;
name += '.';
name += us2fs(seqName.GetNextName());
sync.Set_FilePath(fs2us(name));
if (!outFile.File.Create(name, false))
{
HRESULT res = GetLastError();
AddErrorPath(name);
return res;
}
UInt64 expectSize = volSize;
if (pos < length)
{
const UInt64 rem = length - pos;
if (expectSize > rem)
expectSize = rem;
}
outFile.PreAlloc(expectSize);
}
if (!outFile.Write(buffer, needSize, processedSize))
return GetLastError();
if (needSize != processedSize)
throw g_Message_FileWriteError;
pos += processedSize;
if (outFile.Written == volSize)
{
outFile.Close();
sync.Set_NumFilesCur(++numFiles);
if (volIndex < VolumeSizes.Size())
volIndex++;
}
if (pos - prev >= ((UInt32)1 << 22) || outFile.Written == 0)
{
RINOK(sync.Set_NumBytesCur(pos));
prev = pos;
}
}
}
HRESULT CThreadSplit::ProcessVirt()
{
NIO::CInFile inFile;
if (!inFile.Open(FilePath))
return GetLastError();
NIO::COutFile outFile;
CMyBuffer bufferObject;
if (!bufferObject.Allocate(kBufSize))
return E_OUTOFMEMORY;
Byte *buffer = (Byte *)(void *)bufferObject;
UInt64 curVolSize = 0;
CVolSeqName seqName;
seqName.SetNumDigits(NumVolumes);
UInt64 length;
if (!inFile.GetLength(length))
return GetLastError();
CProgressSync &sync = ProgressDialog.Sync;
sync.Set_NumBytesTotal(length);
UInt64 pos = 0;
UInt64 numFiles = 0;
unsigned volIndex = 0;
for (;;)
{
UInt64 volSize;
if (volIndex < VolumeSizes.Size())
volSize = VolumeSizes[volIndex];
else
volSize = VolumeSizes.Back();
UInt32 needSize = (UInt32)(MyMin((UInt64)kBufSize, volSize - curVolSize));
UInt32 processedSize;
if (!inFile.Read(buffer, needSize, processedSize))
return GetLastError();
if (processedSize == 0)
break;
needSize = processedSize;
if (curVolSize == 0)
{
FString name = VolBasePath;
name += FTEXT('.');
name += us2fs(seqName.GetNextName());
sync.Set_FilePath(fs2us(name));
sync.Set_NumFilesCur(numFiles++);
if (!outFile.Create(name, false))
{
HRESULT res = GetLastError();
AddErrorPath(name);
return res;
}
}
if (!outFile.Write(buffer, needSize, processedSize))
return GetLastError();
if (needSize != processedSize)
throw g_Message_FileWriteError;
curVolSize += processedSize;
if (curVolSize == volSize)
{
outFile.Close();
if (volIndex < VolumeSizes.Size())
volIndex++;
curVolSize = 0;
}
pos += processedSize;
RINOK(sync.Set_NumBytesCur(pos));
}
sync.Set_NumFilesCur(numFiles);
return S_OK;
}