void BlockManager::CreateNew(offset_type blocksize)
{
m_pFile->Seek(offset_type(0), IFile::S_Begin);
m_pFile->Write(&m_Header, offset_type(sizeof(m_Header)));
m_pFile->ReserveSpace(offset_type(sizeof(m_Header)));
}
void BlockManager::FlushHeader()
{
CheckHeader();
m_pFile->Seek(offset_type(0), IFile::S_Begin);
m_pFile->Write(&m_Header, offset_type(sizeof(m_Header)));
}
int main()
{
_CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );
IFile::OpenMode mode = IFile::O_Truncate;
IFile *pFile = OpenDiskFile("test.pkg", mode);
BlockManager* pMgr = new BlockManager(pFile, mode==IFile::O_Truncate, offset_type(1024));
BlockFS *pFS = new BlockFS(pMgr, mode);
pFS->AddFile("test123", "test123", offset_type(sizeof("test123")));
pFS->AddFile("test1234", "test1234", offset_type(sizeof("test1234")));
IFile* pf1 = pFS->OpenFileInPackage("test123");
IFile* pf2 = pFS->OpenFileInPackage("test1234");
assert(pf1->GetSize()==offset_type(sizeof("test123")));
assert(pf2->GetSize()==offset_type(sizeof("test1234")));
char buffer[16];
offset_type r = pf1->Read(buffer, offset_type(sizeof(buffer)));
assert(r == offset_type(sizeof("test123")));
assert(strcmp(buffer, "test123")==0);
r = pf2->Read(buffer, offset_type(sizeof(buffer)));
assert(r == offset_type(sizeof("test1234")));
assert(strcmp(buffer, "test1234")==0);
delete pf1;
delete pf2;
pFS->RemoveFile("test123");
pFS->RemoveFile("test1234");
pFS->AddFile("test1234", "test1234", offset_type(sizeof("test1234")));
delete pFS;
delete pMgr;
delete pFile;
}
BlockManager::BlockManager(IFile* pFile, bool truncate, offset_type blocksize)
:m_pFile(pFile), m_Header(blocksize, offset_type(-1), offset_type(0)), m_pLock(new FileLock)
{
if(truncate)
{
CreateNew(blocksize);
}
else
{
LoadExist(blocksize);
}
}
void BlockManager::WriteEmptyBlockHeader( offset_type blockid, offset_type header )
{
assert(header<blockid&&"Wrong Empty Block header");
assert(blockid < m_Header.Blocks && "out of range");
m_pFile->Seek(CalcOffset(blockid), IFile::S_Begin);
m_pFile->Write(&header, offset_type(sizeof(header)));
}
void BlockManager::ReadEmptyBlockHeader( offset_type blockid, offset_type &header )
{
//FileAutoLock lock(m_pLock);
assert(blockid < m_Header.Blocks && "out of range");
m_pFile->Seek(CalcOffset(blockid), IFile::S_Begin);
m_pFile->Read(&header, offset_type(sizeof(header)));
}
wbtl_file::offset_type wbtl_file::get_next_write_block()
{
// mapping_lock has to be aquired by caller
sortseq::iterator space =
std::find_if(free_space.begin(), free_space.end(),
bind2nd(FirstFit(), write_block_size) _STXXL_FORCE_SEQUENTIAL);
if (space != free_space.end())
{
offset_type region_pos = (*space).first;
offset_type region_size = (*space).second;
free_space.erase(space);
if (region_size > write_block_size)
free_space[region_pos + write_block_size] = region_size - write_block_size;
free_bytes -= write_block_size;
STXXL_VERBOSE_WBTL("wbtl:nextwb p" << FMT_A_S(region_pos, write_block_size) << " F f" << FMT_A_C(free_bytes, free_space.size()));
return region_pos;
}
STXXL_THROW2(io_error, "OutOfSpace, probably fragmented");
return offset_type(-1);
}
int main(int argc, char** argv)
{
_CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );
IFile::OpenMode mode = IFile::O_ReadOnly;
assert(argc>1);
IFile *pFile = OpenDiskFile(argv[1], mode);
BlockManager* pMgr = new BlockManager(pFile, mode==IFile::O_Truncate, offset_type(1024));
BlockFS *pFS = new BlockFS(pMgr, mode);
std::vector<std::string> names;
pFS->ExportFileNames(names);
for (std::vector<std::string>::iterator it = names.begin();
it != names.end(); ++it)
{
IFile* pUnpackedFile = pFS->OpenFileInPackage(it->c_str());
IFile* pTemp = OpenDiskFile(it->c_str(), IFile::O_Truncate);
offset_type length = pUnpackedFile->GetSize();
char* buff = new char[(size_t)length.offset];
printf("Unpacking %s\n", it->c_str());
pUnpackedFile->Read(buff, length);
pTemp->Write(buff, length);
delete []buff;
delete pTemp;
delete pUnpackedFile;
}
delete pFS;
delete pMgr;
delete pFile;
}
__STXXL_BEGIN_NAMESPACE
wbtl_file::wbtl_file(
file * backend_file,
size_type write_buffer_size,
int write_buffers,
int queue_id, int allocator_id) :
disk_queued_file(queue_id, allocator_id), storage(backend_file), sz(0), write_block_size(write_buffer_size),
free_bytes(0), curbuf(1), curpos(write_block_size)
{
assert(write_buffers == 2); // currently hardcoded
write_buffer[0] = static_cast<char *>(stxxl::aligned_alloc<BLOCK_ALIGN>(write_block_size));
write_buffer[1] = static_cast<char *>(stxxl::aligned_alloc<BLOCK_ALIGN>(write_block_size));
buffer_address[0] = offset_type(-1);
buffer_address[1] = offset_type(-1);
}
void BlockManager::LoadExist(offset_type blocksize)
{
m_pFile->Seek(offset_type(0), IFile::S_Begin);
offset_type sizetoread;
sizetoread.offset = sizeof(m_Header);
m_pFile->Read(&m_Header, sizetoread);
assert(m_Header.BlockSize == blocksize && "Block Size Error");
CheckHeader();
}
void wbtl_file::swrite(void* buffer, offset_type offset, size_type bytes)
{
scoped_mutex_lock buffer_lock(buffer_mutex);
// is the block already mapped?
{
scoped_mutex_lock mapping_lock(mapping_mutex);
sortseq::iterator physical = address_mapping.find(offset);
STXXL_VERBOSE_WBTL("wbtl:swrite l" << FMT_A_S(offset, bytes) << " @ <= p" <<
FMT_A_C(physical != address_mapping.end() ? physical->second : 0xffffffff, address_mapping.size()));
if (physical != address_mapping.end()) {
mapping_lock.unlock();
// FIXME: special case if we can replace it in the current writing block
discard(offset, bytes);
}
}
if (bytes > write_block_size - curpos)
{
// not enough space in the current write buffer
if (buffer_address[curbuf] != offset_type(-1)) {
STXXL_VERBOSE_WBTL("wbtl:w2disk p" << FMT_A_S(buffer_address[curbuf], write_block_size));
// mark remaining part as free
if (curpos < write_block_size)
_add_free_region(buffer_address[curbuf] + curpos, write_block_size - curpos);
if (backend_request.get()) {
backend_request->wait(false);
}
backend_request = storage->awrite(write_buffer[curbuf], buffer_address[curbuf], write_block_size);
}
curbuf = 1 - curbuf;
buffer_address[curbuf] = get_next_write_block();
curpos = 0;
}
assert(bytes <= write_block_size - curpos);
// write block into buffer
memcpy(write_buffer[curbuf] + curpos, buffer, bytes);
stats::get_instance()->write_cached(bytes);
scoped_mutex_lock mapping_lock(mapping_mutex);
address_mapping[offset] = buffer_address[curbuf] + curpos;
reverse_mapping[buffer_address[curbuf] + curpos] = place(offset, bytes);
STXXL_VERBOSE_WBTL("wbtl:swrite l" << FMT_A_S(offset, bytes) << " @ => p" << FMT_A_C(buffer_address[curbuf] + curpos, address_mapping.size()));
curpos += bytes;
}
int main(int argc, char* argv[])
{
_CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF );
if(argc < 3)
{
printf("usage: %s directory package", argv[0]);
return 0;
}
IFile::OpenMode mode = IFile::O_Write;
if(!PathFileExistsA(argv[1]))
mode = IFile::O_Truncate;
IFile *pFile = OpenDiskFile(argv[1], mode);
BlockManager* pMgr = new BlockManager(pFile, mode==IFile::O_Truncate, offset_type(1024));
BlockFS *pFS = new BlockFS(pMgr, mode);
std::vector<BlockFileEntry> vecEntries = pFS->GetEntries();
std::map<std::string, MD5Index> mapEntries;
for(std::vector<BlockFileEntry>::iterator it = vecEntries.begin();
it != vecEntries.end(); ++it)
{
if(std::string("")!=it->name)
{
assert(mapEntries.find(it->name)==mapEntries.end());
mapEntries[it->name] = it->index;
}
}
std::vector<std::string> add;
ScanFileRecursively(argv[2], pFS, mapEntries, add);
for(std::map<std::string, MD5Index>::iterator it = mapEntries.begin(); it != mapEntries.end(); ++it)
pFS->RemoveFile(it->first.c_str());
for(std::vector<std::string>::iterator it = add.begin(); it != add.end(); ++it)
{
IFile* pTemp = OpenDiskFile(it->c_str(), IFile::O_ReadOnly);
offset_type length = pTemp->GetSize();
char* buff = new char[(size_t)length.offset];
pTemp->Read(buff, length);
printf("adding %s, %d bytes\n", it->c_str(), length);
pFS->AddFile(it->c_str(), buff, length, length > pFS->GetBlockDataSize());
delete[]buff;
delete pTemp;
}
delete pFS;
delete pMgr;
delete pFile;
return 0;
}
offset_type BlockManager::CalcOffset( offset_type blockid )
{
return offset_type(sizeof(m_Header)+m_Header.BlockSize.offset*blockid.offset);
}
