CASC::FileHandle CASC::OpenFile(StorageHandle const& storage, char const* fileName, DWORD localeMask, bool printErrors /*= false*/)
{
HANDLE handle = nullptr;
if (!::CascOpenFile(storage.get(), fileName, localeMask, 0, &handle))
{
DWORD lastError = GetLastError(); // support checking error set by *Open* call, not the next *Close*
if (printErrors)
fprintf(stderr, "Failed to open '%s' in CASC storage: %s\n", fileName, HumanReadableCASCError(lastError));
CascCloseFile(handle);
SetLastError(lastError);
return FileHandle();
}
return FileHandle(handle);
}
void SharedData::validate(std::ostream &out) const {
Creators cs= get_validators();
for (unsigned int i=0; i< cs.size(); ++i) {
boost::scoped_ptr<Validator>
ptr(cs[i]->create(FileHandle(const_cast<SharedData*>(this))));
ptr->write_errors(out);
}
}
FileHandle FileIndex::openFile(const std::string& filename)
{
auto iterator = files.find( filename );
if( iterator == files.end() )
{
return nullptr;
}
IndexData& f = iterator->second;
bool isArchive = !f.archive.empty();
auto fsName = f.directory + "/" + f.originalName;
char* data = nullptr;
size_t length = 0;
if( isArchive )
{
fsName = f.directory + "/" + f.archive;
LoaderIMG img;
if( ! img.load(fsName) )
{
throw std::runtime_error("Failed to load IMG archive: " + fsName);
}
LoaderIMGFile file;
if( img.findAssetInfo(f.originalName, file) )
{
length = file.size * 2048;
data = img.loadToMemory(f.originalName);
}
}
else
{
std::ifstream dfile(fsName.c_str(), std::ios_base::binary);
if ( ! dfile.is_open()) {
throw std::runtime_error("Unable to open file: " + fsName);
}
dfile.seekg(0, std::ios_base::end);
length = dfile.tellg();
dfile.seekg(0);
data = new char[length];
dfile.read(data, length);
}
if( data == nullptr )
{
return nullptr;
}
return FileHandle( new FileContentsInfo{ data, length } );
}
FileHandle WindowsFiles::internalHandle(const std::string& path) const
{
char modulePath[MAX_PATH] = {0};
char drive[_MAX_DRIVE] = {0};
char dir[_MAX_DIR] = {0};
::GetModuleFileName(NULL, modulePath, MAX_PATH);
_splitpath(modulePath, drive, dir, NULL, NULL);
std::string fullPath = std::string(drive) + dir + path;
return FileHandle(fullPath, Internal);
}
bool FFeaturePackContentSource::LoadPakFileToBuffer(FPakPlatformFile& PakPlatformFile, FString Path, TArray<uint8>& Buffer)
{
bool bResult = false;
TSharedPtr<IFileHandle> FileHandle(PakPlatformFile.OpenRead(*Path));
if( FileHandle.IsValid())
{
Buffer.AddUninitialized(FileHandle->Size());
bResult = FileHandle->Read(Buffer.GetData(), FileHandle->Size());
}
return bResult;
}
bool CopyFileToPak(FArchive& InPak, const FString& InMountPoint, const FPakInputPair& InFile, uint8*& InOutPersistentBuffer, int64& InOutBufferSize, FPakEntryPair& OutNewEntry)
{
TAutoPtr<FArchive> FileHandle(IFileManager::Get().CreateFileReader(*InFile.Source));
bool bFileExists = FileHandle.IsValid();
if (bFileExists)
{
const int64 FileSize = FileHandle->TotalSize();
const int64 PaddedEncryptedFileSize = Align(FileSize, FAES::AESBlockSize);
OutNewEntry.Filename = InFile.Dest.Mid(InMountPoint.Len());
OutNewEntry.Info.Offset = 0; // Don't serialize offsets here.
OutNewEntry.Info.Size = FileSize;
OutNewEntry.Info.UncompressedSize = FileSize;
OutNewEntry.Info.CompressionMethod = COMPRESS_None;
OutNewEntry.Info.bEncrypted = InFile.bNeedEncryption;
if (InOutBufferSize < PaddedEncryptedFileSize)
{
InOutPersistentBuffer = (uint8*)FMemory::Realloc(InOutPersistentBuffer, PaddedEncryptedFileSize);
InOutBufferSize = FileSize;
}
// Load to buffer
FileHandle->Serialize(InOutPersistentBuffer, FileSize);
{
int64 SizeToWrite = FileSize;
if (InFile.bNeedEncryption)
{
for(int64 FillIndex=FileSize; FillIndex < PaddedEncryptedFileSize && InFile.bNeedEncryption; ++FillIndex)
{
// Fill the trailing buffer with random bytes from file
InOutPersistentBuffer[FillIndex] = InOutPersistentBuffer[rand()%FileSize];
}
//Encrypt the buffer before writing it to disk
FAES::EncryptData(InOutPersistentBuffer, PaddedEncryptedFileSize);
// Update the size to be written
SizeToWrite = PaddedEncryptedFileSize;
OutNewEntry.Info.bEncrypted = true;
}
// Calculate the buffer hash value
FSHA1::HashBuffer(InOutPersistentBuffer,FileSize,OutNewEntry.Info.Hash);
// Write to file
OutNewEntry.Info.Serialize(InPak,FPakInfo::PakFile_Version_Latest);
InPak.Serialize(InOutPersistentBuffer,SizeToWrite);
}
}
return bFileExists;
}
bool ShaderManager::loadUseProgram(const LoadUseProgramDescription &desc, vx::StackAllocator* scratchAllocator, std::string* error)
{
// check for not used shader stage
if (strcmp(desc.id, "''") == 0)
{
return true;
}
auto fileHandle = FileHandle(desc.id);
if (!loadProgram(fileHandle, desc.type, *desc.programDir, *desc.includeDir, scratchAllocator, error))
return false;
if (!useProgram(*desc.pipe, fileHandle))
return false;
return true;
}
void WindowsFiles::list(const std::string& path, std::vector< FileHandle>& handles) const
{
handles.clear();
if( isDirectory( path))
{
WIN32_FIND_DATA FileData;
HANDLE hSearch;
std::string strSearchText = path;
strSearchText += "\\*";
hSearch = FindFirstFile( strSearchText.c_str(), &FileData);
do
{
if(!isDots(FileData.cFileName))
handles.push_back( FileHandle( path + "/" + FileData.cFileName));
}
while (FindNextFile(hSearch, &FileData));
FindClose( hSearch);
}
}
File ZeroFileSystem::openFile(const String& path)
{
return FileHandle(1);
}
FileHandle NodeHandle::get_file() const { return FileHandle(shared_); }
FileHandle open_file(const string& filename, const string& mode)
{
return FileHandle(open_file_raw(filename, mode), fclose);
}
bool ExtractFilesFromPak(const TCHAR* InPakFilename, const TCHAR* InDestPath, bool bUseMountPoint = false)
{
FPakFile PakFile(InPakFilename, FParse::Param(FCommandLine::Get(), TEXT("signed")));
if (PakFile.IsValid())
{
FString DestPath(InDestPath);
FArchive& PakReader = *PakFile.GetSharedReader(NULL);
const int64 BufferSize = 8 * 1024 * 1024; // 8MB buffer for extracting
void* Buffer = FMemory::Malloc(BufferSize);
int64 CompressionBufferSize = 0;
uint8* PersistantCompressionBuffer = NULL;
int32 ErrorCount = 0;
int32 FileCount = 0;
FString PakMountPoint = bUseMountPoint ? PakFile.GetMountPoint().Replace( TEXT("../../../"), TEXT("")) : TEXT("");
for (FPakFile::FFileIterator It(PakFile); It; ++It, ++FileCount)
{
const FPakEntry& Entry = It.Info();
PakReader.Seek(Entry.Offset);
uint32 SerializedCrcTest = 0;
FPakEntry EntryInfo;
EntryInfo.Serialize(PakReader, PakFile.GetInfo().Version);
if (EntryInfo == Entry)
{
FString DestFilename(DestPath / PakMountPoint / It.Filename());
TAutoPtr<FArchive> FileHandle(IFileManager::Get().CreateFileWriter(*DestFilename));
if (FileHandle.IsValid())
{
if (Entry.CompressionMethod == COMPRESS_None)
{
BufferedCopyFile(*FileHandle, PakReader, Entry, Buffer, BufferSize);
}
else
{
UncompressCopyFile(*FileHandle, PakReader, Entry, PersistantCompressionBuffer, CompressionBufferSize);
}
UE_LOG(LogPakFile, Display, TEXT("Extracted \"%s\" to \"%s\"."), *It.Filename(), *DestFilename);
}
else
{
UE_LOG(LogPakFile, Error, TEXT("Unable to create file \"%s\"."), *DestFilename);
ErrorCount++;
}
}
else
{
UE_LOG(LogPakFile, Error, TEXT("Serialized hash mismatch for \"%s\"."), *It.Filename());
ErrorCount++;
}
}
FMemory::Free(Buffer);
FMemory::Free(PersistantCompressionBuffer);
UE_LOG(LogPakFile, Error, TEXT("Finished extracting %d files (including %d errors)."), FileCount, ErrorCount);
return true;
}
else
{
UE_LOG(LogPakFile, Error, TEXT("Unable to open pak file \"%s\"."), InPakFilename);
return false;
}
}
bool FCompressedFileBuffer::CompressFileToWorkingBuffer(const FPakInputPair& InFile, uint8*& InOutPersistentBuffer, int64& InOutBufferSize, ECompressionFlags CompressionMethod, const int32 CompressionBlockSize)
{
TAutoPtr<FArchive> FileHandle(IFileManager::Get().CreateFileReader(*InFile.Source));
if(!FileHandle.IsValid())
{
TotalCompressedSize = 0;
return false;
}
Reinitialize(FileHandle.GetOwnedPointer(), CompressionMethod, CompressionBlockSize);
const int64 FileSize = OriginalSize;
const int64 PaddedEncryptedFileSize = Align(FileSize,FAES::AESBlockSize);
if(InOutBufferSize < PaddedEncryptedFileSize)
{
InOutPersistentBuffer = (uint8*)FMemory::Realloc(InOutPersistentBuffer,PaddedEncryptedFileSize);
InOutBufferSize = FileSize;
}
// Load to buffer
FileHandle->Serialize(InOutPersistentBuffer,FileSize);
// Build buffers for working
int64 UncompressedSize = FileSize;
int32 CompressionBufferSize = Align(FCompression::CompressMemoryBound(CompressionMethod,CompressionBlockSize),FAES::AESBlockSize);
EnsureBufferSpace(Align(FCompression::CompressMemoryBound(CompressionMethod,FileSize),FAES::AESBlockSize));
TotalCompressedSize = 0;
int64 UncompressedBytes = 0;
int32 CurrentBlock = 0;
while(UncompressedSize)
{
int32 BlockSize = (int32)FMath::Min<int64>(UncompressedSize,CompressionBlockSize);
int32 CompressedBlockSize = CompressionBufferSize;
FileCompressionBlockSize = FMath::Max<uint32>(BlockSize, FileCompressionBlockSize);
EnsureBufferSpace(TotalCompressedSize+CompressedBlockSize);
if(!FCompression::CompressMemory(CompressionMethod,CompressedBuffer.Get()+TotalCompressedSize,CompressedBlockSize,InOutPersistentBuffer+UncompressedBytes,BlockSize))
{
return false;
}
UncompressedSize -= BlockSize;
UncompressedBytes += BlockSize;
CompressedBlocks[CurrentBlock].CompressedStart = TotalCompressedSize;
CompressedBlocks[CurrentBlock].CompressedEnd = TotalCompressedSize+CompressedBlockSize;
++CurrentBlock;
TotalCompressedSize += CompressedBlockSize;
if(InFile.bNeedEncryption)
{
int32 EncryptionBlockPadding = Align(TotalCompressedSize,FAES::AESBlockSize);
for(int64 FillIndex=TotalCompressedSize; FillIndex < EncryptionBlockPadding; ++FillIndex)
{
// Fill the trailing buffer with random bytes from file
CompressedBuffer.Get()[FillIndex] = CompressedBuffer.Get()[rand()%TotalCompressedSize];
}
TotalCompressedSize += EncryptionBlockPadding - TotalCompressedSize;
}
}
return true;
}
FileHandle create_rmf_buffer(std::string &buffer) {
return FileHandle(internal::create_shared_data_in_buffer(buffer, true));
}
FileHandle create_rmf_file(std::string path) {
return FileHandle(path, true);
}
FileHandle open_rmf_file(std::string path) {
return FileHandle(path, false);
}
void ZipInput::ReadInfos(void *masterFile)
{
// Read infos
m_fileInfos.reserve(m_numberOfFiles);
if (unzGoToFirstFile(static_cast<unzFile>(masterFile)) != UNZ_OK)
{
LogPedantic("Failed to go to first file");
ThrowMsg(Exception::SeekFileFailed, "Failed to seek first file");
}
for (size_t i = 0; i < m_numberOfFiles; ++i)
{
unz_file_pos_s filePos;
if (unzGetFilePos(static_cast<unzFile>(masterFile),
&filePos) != UNZ_OK)
{
LogPedantic("Failed to get file pos");
ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
}
unz_file_info64 fileInfo;
if (unzGetCurrentFileInfo64(static_cast<unzFile>(masterFile),
&fileInfo,
NULL,
0,
NULL,
0,
NULL,
0) != UNZ_OK)
{
LogPedantic("Failed to get file pos");
ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
}
ScopedArray<char> fileName(new char[fileInfo.size_filename + 1]);
ScopedArray<char> fileComment(new char[fileInfo.size_file_comment + 1]);
if (unzGetCurrentFileInfo64(static_cast<unzFile>(masterFile),
&fileInfo,
fileName.Get(),
fileInfo.size_filename + 1,
NULL,
0,
fileComment.Get(),
fileInfo.size_file_comment + 1) != UNZ_OK)
{
LogPedantic("Failed to get file pos");
ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
}
m_fileInfos.push_back(
FileInfo(
FileHandle(
static_cast<size_t>(filePos.pos_in_zip_directory),
static_cast<size_t>(filePos.num_of_file)
),
std::string(fileName.Get()),
std::string(fileComment.Get()),
static_cast<off64_t>(fileInfo.compressed_size),
static_cast<off64_t>(fileInfo.uncompressed_size)
)
);
// If this is not the last file, go to next one
if (i != m_numberOfFiles - 1)
{
if (unzGoToNextFile(
static_cast<unzFile>(masterFile))!= UNZ_OK)
{
LogPedantic("Failed to go to next file");
ThrowMsg(Exception::FileInfoFailed,
"Failed to go to next file");
}
}
}
}
FileHandle FrameHandle::get_file() const {
return FileHandle(get_shared_data());
}
FileHandle WindowsFiles::getFileHandle(const std::string& path, FileType type) const
{
return FileHandle(path, type);
}
FileHandle WindowsFiles::absoluteHandle(const std::string& path) const
{
return FileHandle(path, Absolute);
}
FileHandle WindowsFiles::externalHandle(const std::string& path) const
{
std::string externalStoragePath;
getExternalStoragePath(externalStoragePath);
return FileHandle(externalStoragePath + "/" + path, External);
}
