void DiskCachingFileLoaderCache::InitCache(const std::string &path) {
cacheSize_ = 0;
indexCount_ = 0;
oldestGeneration_ = 0;
maxBlocks_ = MAX_BLOCKS_LOWER_BOUND;
flags_ = 0;
generation_ = 0;
const std::string cacheFilePath = MakeCacheFilePath(path);
bool fileLoaded = LoadCacheFile(cacheFilePath);
// We do some basic locking to protect against two things: crashes and concurrency.
// Concurrency will break the file. Crashes will probably leave it inconsistent.
if (fileLoaded && !LockCacheFile(true)) {
if (RemoveCacheFile(cacheFilePath)) {
// Create a new one.
fileLoaded = false;
} else {
// Couldn't remove, in use? Give up on caching.
CloseFileHandle();
}
}
if (!fileLoaded) {
CreateCacheFile(cacheFilePath);
if (!LockCacheFile(true)) {
CloseFileHandle();
}
}
}
bool DiskCachingFileLoaderCache::LockCacheFile(bool lockStatus) {
if (!f_) {
return false;
}
u32 offset = (u32)offsetof(FileHeader, flags);
bool failed = false;
if (fseek(f_, offset, SEEK_SET) != 0) {
failed = true;
} else if (fread(&flags_, sizeof(u32), 1, f_) != 1) {
failed = true;
}
if (failed) {
ERROR_LOG(LOADER, "Unable to read current flags during disk cache locking");
CloseFileHandle();
return false;
}
// TODO: Also use flock where supported?
if (lockStatus) {
if ((flags_ & FLAG_LOCKED) != 0) {
ERROR_LOG(LOADER, "Could not lock disk cache file for %s", origPath_.c_str());
return false;
}
flags_ |= FLAG_LOCKED;
} else {
if ((flags_ & FLAG_LOCKED) == 0) {
ERROR_LOG(LOADER, "Could not unlock disk cache file for %s", origPath_.c_str());
return false;
}
flags_ &= ~FLAG_LOCKED;
}
if (fseek(f_, offset, SEEK_SET) != 0) {
failed = true;
} else if (fwrite(&flags_, sizeof(u32), 1, f_) != 1) {
failed = true;
} else if (fflush(f_) != 0) {
failed = true;
}
if (failed) {
ERROR_LOG(LOADER, "Unable to write updated flags during disk cache locking");
CloseFileHandle();
return false;
}
if (lockStatus) {
INFO_LOG(LOADER, "Locked disk cache file for %s", origPath_.c_str());
} else {
INFO_LOG(LOADER, "Unlocked disk cache file for %s", origPath_.c_str());
}
return true;
}
void DiskCachingFileLoaderCache::CreateCacheFile(const std::string &path) {
maxBlocks_ = DetermineMaxBlocks();
if (maxBlocks_ < MAX_BLOCKS_LOWER_BOUND) {
GarbageCollectCacheFiles(MAX_BLOCKS_LOWER_BOUND * DEFAULT_BLOCK_SIZE);
maxBlocks_ = DetermineMaxBlocks();
}
if (maxBlocks_ < MAX_BLOCKS_LOWER_BOUND) {
// There's not enough free space to cache, disable.
f_ = nullptr;
ERROR_LOG(LOADER, "Not enough free space; disabling disk cache");
return;
}
flags_ = 0;
f_ = File::OpenCFile(path, "wb+");
if (!f_) {
ERROR_LOG(LOADER, "Could not create disk cache file");
return;
}
#ifdef __ANDROID__
// Android NDK does not support 64-bit file I/O using C streams
fd_ = fileno(f_);
#endif
blockSize_ = DEFAULT_BLOCK_SIZE;
FileHeader header;
memcpy(header.magic, CACHEFILE_MAGIC, sizeof(header.magic));
header.version = CACHE_VERSION;
header.blockSize = blockSize_;
header.filesize = filesize_;
header.maxBlocks = maxBlocks_;
header.flags = flags_;
if (fwrite(&header, sizeof(header), 1, f_) != 1) {
CloseFileHandle();
return;
}
indexCount_ = (filesize_ + blockSize_ - 1) / blockSize_;
index_.clear();
index_.resize(indexCount_);
blockIndexLookup_.resize(maxBlocks_);
memset(&blockIndexLookup_[0], INVALID_INDEX, maxBlocks_ * sizeof(blockIndexLookup_[0]));
if (fwrite(&index_[0], sizeof(BlockInfo), indexCount_, f_) != indexCount_) {
CloseFileHandle();
return;
}
if (fflush(f_) != 0) {
CloseFileHandle();
return;
}
INFO_LOG(LOADER, "Created new disk cache file for %s", origPath_.c_str());
}
void DiskCachingFileLoaderCache::WriteBlockData(BlockInfo &info, u8 *src) {
if (!f_) {
return;
}
s64 blockOffset = GetBlockOffset(info.block);
bool failed = false;
#ifdef __ANDROID__
if (lseek64(fd_, blockOffset, SEEK_SET) != blockOffset) {
failed = true;
} else if (write(fd_, src, blockSize_) != (ssize_t)blockSize_) {
failed = true;
}
#else
if (fseeko(f_, blockOffset, SEEK_SET) != 0) {
failed = true;
} else if (fwrite(src, blockSize_, 1, f_) != 1) {
failed = true;
}
#endif
if (failed) {
ERROR_LOG(LOADER, "Unable to write disk cache data entry.");
CloseFileHandle();
}
}
bool DiskCachingFileLoaderCache::ReadBlockData(u8 *dest, BlockInfo &info, size_t offset, size_t size) {
if (!f_) {
return false;
}
s64 blockOffset = GetBlockOffset(info.block);
// Before we read, make sure the buffers are flushed.
// We might be trying to read an area we've recently written.
fflush(f_);
bool failed = false;
#ifdef __ANDROID__
if (lseek64(fd_, blockOffset, SEEK_SET) != blockOffset) {
failed = true;
} else if (read(fd_, dest + offset, size) != (ssize_t)size) {
failed = true;
}
#else
if (fseeko(f_, blockOffset, SEEK_SET) != 0) {
failed = true;
} else if (fread(dest + offset, size, 1, f_) != 1) {
failed = true;
}
#endif
if (failed) {
ERROR_LOG(LOADER, "Unable to read disk cache data entry.");
CloseFileHandle();
}
return !failed;
}
void DiskCachingFileLoaderCache::LoadCacheIndex() {
if (fseek(f_, sizeof(FileHeader), SEEK_SET) != 0) {
CloseFileHandle();
return;
}
indexCount_ = (filesize_ + blockSize_ - 1) / blockSize_;
index_.resize(indexCount_);
blockIndexLookup_.resize(maxBlocks_);
memset(&blockIndexLookup_[0], INVALID_INDEX, maxBlocks_ * sizeof(blockIndexLookup_[0]));
if (fread(&index_[0], sizeof(BlockInfo), indexCount_, f_) != indexCount_) {
CloseFileHandle();
return;
}
// Now let's set some values we need.
oldestGeneration_ = std::numeric_limits<u16>::max();
generation_ = 0;
cacheSize_ = 0;
for (size_t i = 0; i < index_.size(); ++i) {
if (index_[i].block > maxBlocks_) {
index_[i].block = INVALID_BLOCK;
}
if (index_[i].block == INVALID_BLOCK) {
continue;
}
if (index_[i].generation < oldestGeneration_) {
oldestGeneration_ = index_[i].generation;
}
if (index_[i].generation > generation_) {
generation_ = index_[i].generation;
}
++cacheSize_;
blockIndexLookup_[index_[i].block] = (u32)i;
}
}
void DiskCachingFileLoaderCache::WriteIndexData(u32 indexPos, BlockInfo &info) {
if (!f_) {
return;
}
u32 offset = (u32)sizeof(FileHeader) + indexPos * (u32)sizeof(BlockInfo);
bool failed = false;
if (fseek(f_, offset, SEEK_SET) != 0) {
failed = true;
} else if (fwrite(&info, sizeof(BlockInfo), 1, f_) != 1) {
failed = true;
}
if (failed) {
ERROR_LOG(LOADER, "Unable to write disk cache index entry.");
CloseFileHandle();
}
}
void DiskCachingFileLoaderCache::ShutdownCache() {
if (f_) {
bool failed = false;
if (fseek(f_, sizeof(FileHeader), SEEK_SET) != 0) {
failed = true;
} else if (fwrite(&index_[0], sizeof(BlockInfo), indexCount_, f_) != indexCount_) {
failed = true;
} else if (fflush(f_) != 0) {
failed = true;
}
if (failed) {
// Leave it locked, it's broken.
ERROR_LOG(LOADER, "Unable to flush disk cache.");
} else {
LockCacheFile(false);
}
CloseFileHandle();
}
index_.clear();
blockIndexLookup_.clear();
cacheSize_ = 0;
}
/**
* @brief Close file view pane and close file handle.
* @param hwnd - parent window handle.
*/
inline static void CloseViewPane(HWND hwnd)
{
CloseFileView(hwnd);
CloseFileHandle();
}
bool MAPFAM::OpenTableFile(PGLOBAL g)
{
char filename[_MAX_PATH];
int len;
MODE mode = Tdbp->GetMode();
PFBLOCK fp;
PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr;
#if defined(_DEBUG)
// Insert mode is no more handled using file mapping
assert(mode != MODE_INSERT);
#endif // _DEBUG
/*********************************************************************/
/* We used the file name relative to recorded datapath. */
/*********************************************************************/
PlugSetPath(filename, To_File, Tdbp->GetPath());
/*********************************************************************/
/* Under Win32 the whole file will be mapped so we can use it as */
/* if it were entirely read into virtual memory. */
/* Firstly we check whether this file have been already mapped. */
/*********************************************************************/
if (mode == MODE_READ) {
for (fp = dbuserp->Openlist; fp; fp = fp->Next)
if (fp->Type == TYPE_FB_MAP && !stricmp(fp->Fname, filename)
&& fp->Count && fp->Mode == mode)
break;
if (trace)
htrc("Mapping file, fp=%p\n", fp);
} else
fp = NULL;
if (fp) {
/*******************************************************************/
/* File already mapped. Just increment use count and get pointer. */
/*******************************************************************/
fp->Count++;
Memory = fp->Memory;
len = fp->Length;
} else {
/*******************************************************************/
/* If required, delete the whole file if no filtering is implied. */
/*******************************************************************/
bool del;
HANDLE hFile;
MEMMAP mm;
del = mode == MODE_DELETE && !Tdbp->GetNext();
if (del)
DelRows = Cardinality(g);
/*******************************************************************/
/* Create the mapping file object. */
/*******************************************************************/
hFile = CreateFileMap(g, filename, &mm, mode, del);
if (hFile == INVALID_HANDLE_VALUE) {
DWORD rc = GetLastError();
if (!(*g->Message))
sprintf(g->Message, MSG(OPEN_MODE_ERROR),
"map", (int) rc, filename);
if (trace)
htrc("CreateFileMap: %s\n", g->Message);
return (mode == MODE_READ && rc == ENOENT)
? PushWarning(g, Tdbp) : true;
} // endif hFile
/*******************************************************************/
/* Get the file size (assuming file is smaller than 4 GB) */
/*******************************************************************/
len = mm.lenL;
Memory = (char *)mm.memory;
if (!len) { // Empty or deleted file
CloseFileHandle(hFile);
Tdbp->ResetSize();
return false;
} // endif len
if (!Memory) {
CloseFileHandle(hFile);
sprintf(g->Message, MSG(MAP_VIEW_ERROR),
filename, GetLastError());
return true;
} // endif Memory
#if defined(WIN32)
if (mode != MODE_DELETE) {
#else // !WIN32
if (mode == MODE_READ) {
#endif // !WIN32
CloseFileHandle(hFile); // Not used anymore
hFile = INVALID_HANDLE_VALUE; // For Fblock
} // endif Mode
/*******************************************************************/
/* Link a Fblock. This make possible to reuse already opened maps */
/* and also to automatically unmap them in case of error g->jump. */
/* Note: block can already exist for previously closed file. */
/*******************************************************************/
fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK));
fp->Type = TYPE_FB_MAP;
fp->Fname = (char*)PlugSubAlloc(g, NULL, strlen(filename) + 1);
strcpy((char*)fp->Fname, filename);
fp->Next = dbuserp->Openlist;
dbuserp->Openlist = fp;
fp->Count = 1;
fp->Length = len;
fp->Memory = Memory;
fp->Mode = mode;
fp->File = NULL;
fp->Handle = hFile; // Used for Delete
} // endif fp
To_Fb = fp; // Useful when closing
/*********************************************************************/
/* The pseudo "buffer" is here the entire file mapping view. */
/*********************************************************************/
Fpos = Mempos = Memory;
Top = Memory + len;
if (trace)
htrc("fp=%p count=%d MapView=%p len=%d Top=%p\n",
fp, fp->Count, Memory, len, Top);
return AllocateBuffer(g); // Useful for DBF files
} // end of OpenTableFile
/***********************************************************************/
/* GetRowID: return the RowID of last read record. */
/***********************************************************************/
int MAPFAM::GetRowID(void)
{
return Rows;
} // end of GetRowID
/***********************************************************************/
/* GetPos: return the position of last read record. */
/***********************************************************************/
int MAPFAM::GetPos(void)
{
return Fpos - Memory;
} // end of GetPos
/***********************************************************************/
/* GetNextPos: return the position of next record. */
/***********************************************************************/
int MAPFAM::GetNextPos(void)
{
return Mempos - Memory;
} // end of GetNextPos
/***********************************************************************/
/* SetPos: Replace the table at the specified position. */
/***********************************************************************/
bool MAPFAM::SetPos(PGLOBAL g, int pos)
{
Fpos = Mempos = Memory + pos;
if (Mempos >= Top || Mempos < Memory) {
strcpy(g->Message, MSG(INV_MAP_POS));
return true;
} // endif Mempos
Placed = true;
return false;
} // end of SetPos
bool DiskCachingFileLoaderCache::RemoveCacheFile(const std::string &path) {
// Note that some platforms, you can't delete open files. So we check.
CloseFileHandle();
return File::Delete(path);
}