DEHT * create_empty_DEHT(const char *prefix,
hashKeyIntoTableFunctionPtr hashfun, hashKeyforEfficientComparisonFunctionPtr validfun,
const char *dictName,
int numEntriesInHashTable, int nPairsPerBlock, int nBytesPerKey,
int nUserBytes)
{
bool_t errorState = FALSE;
DEHT * ht = NULL;
TRACE_FUNC_ENTRY();
/* sanity */
CHECK(0 != numEntriesInHashTable);
CHECK(0 != nPairsPerBlock);
CHECK(0 != nBytesPerKey);
ht = DEHT_initInstance(prefix, "c+b", hashfun, validfun);
CHECK(NULL != ht);
/* Do extra inits */
ht->header.magic = DEHT_HEADER_MAGIC;
SAFE_STRNCPY(ht->header.sDictionaryName, dictName, sizeof(ht->header.sDictionaryName));
ht->header.numEntriesInHashTable = numEntriesInHashTable;
ht->header.nPairsPerBlock = nPairsPerBlock;
ht->header.nBytesPerValidationKey = nBytesPerKey;
ht->header.numUnrelatedBytesSaved = nUserBytes;
/* write header to disk */
CHECK_MSG(ht->sKeyfileName, (1 == fwrite(&(ht->header), sizeof(ht->header), 1, ht->keyFP)));
/* write (empty) pointer table to disk */
CHECK_MSG(ht->sKeyfileName, (growFile(ht->keyFP, sizeof(DEHT_DISK_PTR) * numEntriesInHashTable)));
/* write (empty) user data to disk */
CHECK_MSG(ht->sKeyfileName, (growFile(ht->dataFP, ht->header.numUnrelatedBytesSaved)));
goto LBL_CLEANUP;
LBL_ERROR:
errorState = TRUE;
TRACE_FUNC_ERROR();
LBL_CLEANUP:
if (errorState) {
if (NULL != ht) {
DEHT_freeResources(ht, TRUE);
ht = NULL;
}
}
TRACE_FUNC_EXIT();
return ht;
}
bool MemoryStream::seek(int offset, uint from)
{
Assert(from == SEEK_SET || from == SEEK_CUR || from == SEEK_END);
int64 newPos = m_pos;
if (from == SEEK_SET)
newPos = offset;
else if (from == SEEK_CUR)
newPos += offset;
else if (from == SEEK_END)
newPos = m_fileSize + offset;
else
return false;
if (newPos < 0) // bad seek
return false;
if (static_cast<uint>(newPos) > m_fileSize) // 超过文件大小
{
if (m_growBytes == 0)
return false;
growFile((uint)newPos);
}
m_pos = (uint)newPos;
return true;
}
bool MemoryStream::write(const void* buffer, uint toWrite)
{
Assert(buffer != NULL);
if (buffer == NULL || toWrite == 0)
return false;
if (m_pos + toWrite > m_bufSize)
{
if (m_growBytes == 0)
return false;
growFile(m_pos + toWrite);
}
Assert(m_pos + toWrite <= m_bufSize); // growFile success ?
__memcpy((uchar*)m_buffer + m_pos, (uchar*)buffer, toWrite);
m_pos += toWrite;
if (m_pos > m_fileSize)
m_fileSize = m_pos;
return true;
}
Uint32 MMapFile::write(const void* buf, Uint32 buf_size)
{
if (!fptr || mode == QIODevice::ReadOnly)
return 0;
// check if data fits in memory mapping
if (ptr + buf_size > size)
throw Error(i18n("Cannot write beyond end of the mmap buffer."));
Out(SYS_GEN | LOG_DEBUG) << "MMapFile::write : " << (ptr + buf_size) << " " << file_size << endl;
// enlarge the file if necessary
if (ptr + buf_size > file_size)
{
growFile(ptr + buf_size);
}
// memcpy data
memcpy(&data[ptr], buf, buf_size);
// update ptr
ptr += buf_size;
// update file size if necessary
if (ptr >= size)
size = ptr;
return buf_size;
}
static DEHT_DISK_PTR DEHT_allocKeyBlock(DEHT * ht)
{
DEHT_DISK_PTR newBlock = 0;
TRACE_FUNC_ENTRY();
CHECK(NULL != ht);
/* this is the first block - use the end of the file */
(void) fflush(ht->keyFP);
CHECK(0 == fseek(ht->keyFP, 0, SEEK_END));
newBlock = ftell(ht->keyFP);
/* alloc an empty block (init to NULLs) */
CHECK(growFile(ht->keyFP, KEY_FILE_BLOCK_SIZE(ht)));
TRACE_FPRINTF((stderr, "TRACE: %s:%d (%s): allocated a block at %#x\n", __FILE__, __LINE__, __FUNCTION__, (uint_t) newBlock));
goto LBL_CLEANUP;
LBL_ERROR:
newBlock = 0;
TRACE_FUNC_ERROR();
LBL_CLEANUP:
TRACE_FUNC_EXIT();
return newBlock;
}
void CacheFile::write(const Uint8* buf,Uint32 size,Uint64 off)
{
QMutexLocker lock(&mutex);
bool close_again = false;
// reopen the file if necessary
if (fd == -1)
{
// Out() << "Reopening " << path << endl;
openFile(RW);
close_again = true;
}
if (read_only)
throw Error(i18n("Cannot open %1 for writing : readonly filesystem").arg(path));
if (off + size > max_size)
{
Out() << "Warning : writing past the end of " << path << endl;
Out() << (off + size) << " " << max_size << endl;
}
if (file_size < off)
{
//Out() << QString("Writing %1 bytes at %2").arg(size).arg(off) << endl;
growFile(off - file_size);
}
// jump to right position
SeekFile(fd,(Int64)off,SEEK_SET);
int ret = ::write(fd,buf,size);
if (close_again)
closeTemporary();
if (ret == -1)
throw Error(i18n("Error writing to %1 : %2").arg(path).arg(strerror(errno)));
else if ((Uint32)ret != size)
{
Out() << QString("Incomplete write of %1 bytes, should be %2").arg(ret).arg(size) << endl;
throw Error(i18n("Error writing to %1").arg(path));
}
if (off + size > file_size)
file_size = off + size;
}
void CacheFile::write(const Uint8* buf,Uint32 size,Uint64 off)
{
QMutexLocker lock(&mutex);
bool close_again = false;
// reopen the file if necessary
if (!fptr)
{
// Out(SYS_DIO|LOG_DEBUG) << "Reopening " << path << endl;
openFile(RW);
close_again = true;
}
if (read_only)
throw Error(i18n("Cannot open %1 for writing: readonly filesystem",path));
if (off + size > max_size)
{
Out(SYS_DIO|LOG_DEBUG) << "Warning : writing past the end of " << path << endl;
Out(SYS_DIO|LOG_DEBUG) << (off + size) << " " << max_size << endl;
throw Error(i18n("Attempting to write beyond the maximum size of %1",path));
}
if (file_size < off)
{
//Out(SYS_DIO|LOG_DEBUG) << QString("Writing %1 bytes at %2").arg(size).arg(off) << endl;
growFile(off - file_size);
}
// jump to right position
if (!fptr->seek(off))
throw Error(i18n("Failed to seek file %1: %2",path,fptr->errorString()));
if (fptr->write((const char*)buf,size) != size)
{
throw Error(i18n("Failed to write to file %1: %2",path,fptr->errorString()));
}
if (close_again)
closeTemporary();
if (off + size > file_size)
file_size = off + size;
}
bool MemoryStream::setLength(uint newLen)
{
Assert(newLen <= 0xffffffffUL);
if (newLen > m_bufSize)
{
if (m_growBytes == 0)
return false;
growFile(newLen);
}
if (newLen < m_pos)
m_pos = newLen;
m_fileSize = newLen;
return true;
}
void* CacheFile::map(MMappeable* thing,Uint64 off,Uint32 size,Mode mode)
{
QMutexLocker lock(&mutex);
// reopen the file if necessary
if (fd == -1)
{
// Out() << "Reopening " << path << endl;
openFile(mode);
}
if (read_only && mode != READ)
{
throw Error(i18n("Cannot open %1 for writing : readonly filesystem").arg(path));
}
if (off + size > max_size)
{
Out() << "Warning : writing past the end of " << path << endl;
Out() << (off + size) << " " << max_size << endl;
return 0;
}
int mmap_flag = 0;
switch (mode)
{
case READ:
mmap_flag = PROT_READ;
break;
case WRITE:
mmap_flag = PROT_WRITE;
break;
case RW:
mmap_flag = PROT_READ|PROT_WRITE;
break;
}
if (off + size > file_size)
{
Uint64 to_write = (off + size) - file_size;
// Out() << "Growing file with " << to_write << " bytes" << endl;
growFile(to_write);
}
Uint32 page_size = sysconf(_SC_PAGESIZE);
if (off % page_size > 0)
{
// off is not a multiple of the page_size
// so we play around a bit
Uint32 diff = (off % page_size);
Uint64 noff = off - diff;
// Out() << "Offsetted mmap : " << diff << endl;
#if HAVE_MMAP64
char* ptr = (char*)mmap64(0, size + diff, mmap_flag, MAP_SHARED, fd, noff);
#else
char* ptr = (char*)mmap(0, size + diff, mmap_flag, MAP_SHARED, fd, noff);
#endif
if (ptr == MAP_FAILED)
{
Out() << "mmap failed : " << QString(strerror(errno)) << endl;
return 0;
}
else
{
CacheFile::Entry e;
e.thing = thing;
e.offset = off;
e.diff = diff;
e.ptr = ptr;
e.size = size + diff;
e.mode = mode;
mappings.insert((void*)(ptr + diff),e);
return ptr + diff;
}
}
else
{
#if HAVE_MMAP64
void* ptr = mmap64(0, size, mmap_flag, MAP_SHARED, fd, off);
#else
void* ptr = mmap(0, size, mmap_flag, MAP_SHARED, fd, off);
#endif
if (ptr == MAP_FAILED)
{
Out() << "mmap failed : " << QString(strerror(errno)) << endl;
return 0;
}
else
{
CacheFile::Entry e;
e.thing = thing;
e.offset = off;
e.ptr = ptr;
e.diff = 0;
e.size = size;
e.mode = mode;
mappings.insert(ptr,e);
return ptr;
}
}
}
void* CacheFile::map(MMappeable* thing,Uint64 off,Uint32 size,Mode mode)
{
QMutexLocker lock(&mutex);
// reopen the file if necessary
if (!fptr)
{
// Out(SYS_DIO|LOG_DEBUG) << "Reopening " << path << endl;
QStorageInfo mount(path); //ntfs cannot handle mmap properly
if (!OpenFileAllowed() || mount.fileSystemType()=="fuseblk" || mount.fileSystemType().startsWith("ntfs"))
return 0; // Running out of file descriptors, force buffered mode
openFile(mode);
}
if (read_only && mode != READ)
{
throw Error(i18n("Cannot open %1 for writing: readonly filesystem",path));
}
if (off + size > max_size)
{
Out(SYS_DIO|LOG_DEBUG) << "Warning : writing past the end of " << path << endl;
Out(SYS_DIO|LOG_DEBUG) << (off + size) << " " << max_size << endl;
throw Error(i18n("Attempting to write beyond the maximum size of %1",path));
}
/*
if (!read_only && (mode == WRITE || mode == RW) && !allocateBytes(off,size))
throw Error(i18n("Not enough free disk space for %1",path));
*/
int mmap_flag = 0;
switch (mode)
{
case READ:
mmap_flag = PROT_READ;
break;
case WRITE:
mmap_flag = PROT_WRITE;
break;
case RW:
mmap_flag = PROT_READ|PROT_WRITE;
break;
}
if (off + size > file_size)
{
Uint64 to_write = (off + size) - file_size;
// Out(SYS_DIO|LOG_DEBUG) << "Growing file with " << to_write << " bytes" << endl;
growFile(to_write);
}
#ifndef Q_WS_WIN
int fd = fptr->handle();
Uint32 page_size = sysconf(_SC_PAGESIZE);
if (off % page_size > 0)
{
// off is not a multiple of the page_size
// so we play around a bit
Uint32 diff = (off % page_size);
Uint64 noff = off - diff;
// Out(SYS_DIO|LOG_DEBUG) << "Offsetted mmap : " << diff << endl;
#ifdef HAVE_MMAP64
char* ptr = (char*)mmap64(0, size + diff, mmap_flag, MAP_SHARED, fd, noff);
#else
char* ptr = (char*)mmap(0, size + diff, mmap_flag, MAP_SHARED, fd, noff);
#endif
if (ptr == MAP_FAILED)
{
Out(SYS_DIO|LOG_DEBUG) << "mmap failed : " << QString(strerror(errno)) << endl;
return 0;
}
else
{
CacheFile::Entry e;
e.thing = thing;
e.offset = off;
e.diff = diff;
e.ptr = ptr;
e.size = size + diff;
e.mode = mode;
mappings.insert((void*)(ptr + diff),e);
return ptr + diff;
}
}
else
{
#ifdef HAVE_MMAP64
void* ptr = mmap64(0, size, mmap_flag, MAP_SHARED, fd, off);
#else
void* ptr = mmap(0, size, mmap_flag, MAP_SHARED, fd, off);
#endif
if (ptr == MAP_FAILED)
{
Out(SYS_DIO|LOG_DEBUG) << "mmap failed : " << QString(strerror(errno)) << endl;
return 0;
}
else
{
CacheFile::Entry e;
e.thing = thing;
e.offset = off;
e.ptr = ptr;
e.diff = 0;
e.size = size;
e.mode = mode;
mappings.insert(ptr,e);
return ptr;
}
}
#else // Q_WS_WIN
char* ptr = (char*)fptr->map(off,size);
if (!ptr)
{
Out(SYS_DIO|LOG_DEBUG) << "mmap failed3 : " << fptr->handle() << " " << QString(strerror(errno)) << endl;
Out(SYS_DIO|LOG_DEBUG) << off << " " << size << endl;
return 0;
}
else
{
CacheFile::Entry e;
e.thing = thing;
e.offset = off;
e.ptr = ptr;
e.diff = 0;
e.size = size;
e.mode = mode;
mappings.insert(ptr,e);
return ptr;
}
#endif
}
/*
* Write data to current position in file.
* Return > 0 on success, < 0 on failure.
*/
int GOSFS_Write(struct File *file, void *buf, ulong_t numBytes) {
GOSFSinstance *instance = (GOSFSinstance *) file->mountPoint->fsData;
GOSFSptr *filePtr = (GOSFSptr *) file->fsData;
GOSFSdirectory *dir;
ulong_t start = file->filePos, endPos = file->endPos;
ulong_t seekPos, vblock, offset, length;
int rc, physBlock;
char *blockBuf;
/* Do we even have write access? Is numBytes valid? */
if (!(file->mode & O_WRITE) || numBytes > INT_MAX
|| filePtr->node.isDirectory)
return EINVALID;
blockBuf = Malloc(GOSFS_BLOCK_SIZE);
if (!blockBuf)
goto memfail;
seekPos = start;
while (numBytes) {
vblock = seekPos / GOSFS_BLOCK_SIZE;
offset = seekPos % GOSFS_BLOCK_SIZE;
length = GOSFS_BLOCK_SIZE - offset;
if (numBytes < length)
length = numBytes;
/* Do we need to allocate a new block? */
if (offset == 0 && seekPos >= file->endPos) {
/* We should grow the file */
physBlock = growFile(file);
if (physBlock < 0)
goto fail;
} else {
physBlock = getPhysicalBlockNum(instance, &filePtr->node, vblock);
}
/* Next read in block into buffer */
rc = readGOSFSBlock(instance->dev, physBlock, blockBuf,
GOSFS_BLOCK_SIZE);
if (rc < 0)
goto fail;
/* Memcpy() over to replace buffer data */
memcpy(blockBuf + offset, buf, length);
/* Write out buffer back to disk */
rc = writeGOSFSBlock(instance->dev, physBlock, blockBuf,
GOSFS_BLOCK_SIZE);
if (rc < 0)
goto fail;
/* Increment position */
seekPos += length;
buf += length;
numBytes -= length;
if (seekPos > file->endPos)
file->endPos = seekPos;
}
/* Update the file position */
file->filePos = seekPos;
/* We may need to update the file size */
if (endPos < file->endPos) {
filePtr->node.size = file->endPos;
dir = (GOSFSdirectory *) blockBuf;
/* Now update file node on disk */
rc = readGOSFSBlock(instance->dev, filePtr->blockNum, dir,
sizeof(GOSFSdirectory));
if (rc < 0)
goto fail;
memcpy(&dir->files[filePtr->offset], &filePtr->node,
sizeof(GOSFSfileNode));
rc = writeGOSFSBlock(instance->dev, filePtr->blockNum, dir,
sizeof(GOSFSdirectory));
}
Free(blockBuf);
return seekPos - start;
memfail: rc = ENOMEM;
goto fail;
fail: if (blockBuf)
Free(blockBuf);
return rc;
}
