void BitmapBuffer::save()
{
//loadfromtemp();
static bool first = true;
string filename = dir + "/BitmapBuffer";
MMapBuffer * buffer; //new MMapBuffer(filename.c_str(), 0);
string predicateFile(filename);
predicateFile.append("_predicate");
MMapBuffer * predicateBuffer =new MMapBuffer(predicateFile.c_str(), predicate_managers[0].size() * (sizeof(ID) + sizeof(size_t)) * 2);
char* predicateWriter = predicateBuffer->get_address();
char* bufferWriter = NULL;
map<ID, ChunkManager*>::const_iterator iter = predicate_managers[0].begin();
char* startPtr;
size_t offset = 0;
startPtr = iter->second->ptrs[0];
if(first == true) {
buffer = new MMapBuffer(filename.c_str(), iter->second->meta->length[0]);
//offset = buffer->get_offset();
//first = false;
} else {
//buffer = new MMapBuffer(filename.c_str(), iter->second->meta->length[0], true);
//offset = buffer->get_offset();
}
predicateWriter = predicateBuffer->get_address();
bufferWriter = buffer->get_address();
vector<size_t>::iterator pageNoIter = iter->second->usedPage[0].begin(),
limit = iter->second->usedPage[0].end();
for(; pageNoIter != limit; pageNoIter++ ) {
size_t pageNo = *pageNoIter;
memcpy(bufferWriter, temp1->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
bufferWriter = bufferWriter + MemoryBuffer::pagesize;
}
*((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID);
*((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t);
offset = offset + iter->second->meta->length[0];
bufferWriter = buffer->resize(iter->second->meta->length[1]);
char* startPos = bufferWriter + offset;
pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end();
for(; pageNoIter != limit; pageNoIter++ ) {
size_t pageNo = *pageNoIter;
memcpy(startPos, temp2->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
startPos = startPos + MemoryBuffer::pagesize;
}
assert(iter->second->meta->length[1] == iter->second->usedPage[1].size() * MemoryBuffer::pagesize);
offset = offset + iter->second->meta->length[1];
iter++;
for(; iter != predicate_managers[0].end(); iter++) {
bufferWriter = buffer->resize(iter->second->meta->length[0]);
startPos = bufferWriter + offset;
pageNoIter = iter->second->usedPage[0].begin(); limit = iter->second->usedPage[0].end();
for(; pageNoIter != limit; pageNoIter++) {
size_t pageNo = *pageNoIter;
memcpy(startPos, temp1->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
startPos = startPos + MemoryBuffer::pagesize;
}
//cout<<"used page count: "<<iter->second->usedPage[0].size()<<endl;
//iter->second->meta->endPtr[0] = startPos + iter->second->meta->usedSpace[0]; //used to build index;
*((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID);
*((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t);
offset += iter->second->meta->length[0];
assert(iter->second->usedPage[0].size() * MemoryBuffer::pagesize == iter->second->meta->length[0]);
bufferWriter = buffer->resize(iter->second->meta->length[1]);
startPos = bufferWriter + offset;
//iter->second->meta->startPtr[1] = startPos; //used to build index;
//iter->second->meta->endPtr[1] = startPos + iter->second->meta->usedSpace[1];
pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end();
for(; pageNoIter != limit; pageNoIter++) {
size_t pageNo = *pageNoIter;
memcpy(startPos, temp2->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
startPos = startPos + MemoryBuffer::pagesize;
}
offset += iter->second->meta->length[1];
assert(iter->second->usedPage[1].size() * MemoryBuffer::pagesize == iter->second->meta->length[1]);
}
buffer->flush();
temp1->discard();
temp2->discard();
iter = predicate_managers[1].begin();
for(; iter != predicate_managers[1].end(); iter++) {
bufferWriter = buffer->resize(iter->second->meta->length[0]);
startPos = bufferWriter + offset;
pageNoIter = iter->second->usedPage[0].begin(); limit = iter->second->usedPage[0].end();
for(; pageNoIter != limit; pageNoIter++) {
size_t pageNo = *pageNoIter;
memcpy(startPos, temp3->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
startPos = startPos + MemoryBuffer::pagesize;
}
*((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID);
*((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t);
offset += iter->second->meta->length[0];
assert(iter->second->usedPage[0].size() * MemoryBuffer::pagesize == iter->second->meta->length[0]);
bufferWriter = buffer->resize(iter->second->usedPage[1].size() * MemoryBuffer::pagesize);
startPos = bufferWriter + offset;
pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end();
for(; pageNoIter != limit; pageNoIter++) {
size_t pageNo = *pageNoIter;
memcpy(startPos, temp4->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize);
startPos = startPos + MemoryBuffer::pagesize;
}
offset += iter->second->meta->length[1];
assert(iter->second->usedPage[1].size() * MemoryBuffer::pagesize == iter->second->meta->length[1]);
}
buffer->flush();
predicateBuffer->flush();
predicateWriter = predicateBuffer->get_address();
int i = 0;
ID id;
for(iter = predicate_managers[0].begin(); iter != predicate_managers[0].end(); iter++, i++) {
id = *((ID*)predicateWriter);
assert(iter->first == id);
predicateWriter = predicateWriter + sizeof(ID);
offset = *((size_t*)predicateWriter);
predicateWriter = predicateWriter + sizeof(size_t);
char* base = buffer->get_address() + offset;
iter->second->meta = (ChunkManagerMeta*)base;
iter->second->meta->startPtr[0] = base + sizeof(ChunkManagerMeta);
iter->second->meta->endPtr[0] = iter->second->meta->startPtr[0] + iter->second->meta->usedSpace[0];
iter->second->meta->startPtr[1] = base + iter->second->meta->length[0];
iter->second->meta->endPtr[1] = iter->second->meta->startPtr[1] + iter->second->meta->usedSpace[1];
//::printMeta(*(iter->second->meta));
}
for(iter = predicate_managers[1].begin(); iter != predicate_managers[1].end(); iter++, i++) {
id = *((ID*)predicateWriter);
assert(iter->first == id);
predicateWriter = predicateWriter + sizeof(ID);
offset = *((size_t*)predicateWriter);
predicateWriter = predicateWriter + sizeof(size_t);
char* base = buffer->get_address() + offset;
iter->second->meta = (ChunkManagerMeta*)base;
iter->second->meta->startPtr[0] = base + sizeof(ChunkManagerMeta);
iter->second->meta->endPtr[0] = iter->second->meta->startPtr[0] + iter->second->meta->usedSpace[0];
iter->second->meta->startPtr[1] = base + iter->second->meta->length[0];
iter->second->meta->endPtr[1] = iter->second->meta->startPtr[1] + iter->second->meta->usedSpace[1];
//::printMeta(*(iter->second->meta));
}
temp3->discard();
temp4->discard();
//build index;
MMapBuffer* bitmapIndex = NULL;
#ifdef DEBUG
cout<<"build hash index for subject"<<endl;
#endif
for ( map<ID,ChunkManager*>::iterator iter = predicate_managers[0].begin(); iter != predicate_managers[0].end(); iter++ ) {
if ( iter->second != NULL ) {
#ifdef DEBUG
cout<<iter->first<<endl;
#endif
iter->second->buildChunkIndex();
iter->second->getChunkIndex(1)->save(bitmapIndex);
iter->second->getChunkIndex(2)->save(bitmapIndex);
}
}
#ifdef DEBUG
cout<<"build hash index for object"<<endl;
#endif
for ( map<ID, ChunkManager*>::iterator iter = predicate_managers[1].begin(); iter != predicate_managers[1].end(); iter++ ) {
if ( iter->second != NULL ) {
#ifdef DEBUF
cout<<iter->first<<endl;
#endif
iter->second->buildChunkIndex();
iter->second->getChunkIndex(1)->save(bitmapIndex);
iter->second->getChunkIndex(2)->save(bitmapIndex);
}
}
delete bitmapIndex;
delete buffer;
delete predicateBuffer;
}
TripleBitRepository* TripleBitRepository::create(const string path)
{
TripleBitRepository* repo = new TripleBitRepository();
string filename = path + "/BitmapBuffer";
if(OSFile::fileExists(filename) == false) {
//file dose not exist, repository has not been build;
fprintf(stderr, "database files dose not exist!");
return NULL;
}
// load the repository from image files;
//load bitmap
#ifdef DEBUG
cout<<filename.c_str()<<endl;
#endif
repo->bitmapImage = new MMapBuffer(filename.c_str(), 0);
string predicateFile(filename);
predicateFile.append("_predicate");
string indexFile(filename);
indexFile.append("_index");
repo->bitmapPredicateImage = new MMapBuffer(predicateFile.c_str(), 0);
repo->bitmapIndexImage = new MMapBuffer(indexFile.c_str(), 0);
repo->bitmapBuffer = BitmapBuffer::load(repo->bitmapImage, repo->bitmapIndexImage, repo->bitmapPredicateImage);
repo->UriTable = URITable::load(path);
repo->preTable = PredicateTable::load(path);
#ifdef DEBUG
cout<<"total triple count: "<<repo->bitmapBuffer->getTripleCount()<<endl;
cout<<"URITableSize: "<<repo->UriTable->getSize()<<endl;
cout<<"predicateTableSize: "<<repo->preTable->getSize()<<endl;
#endif
// cout<<"begin load statistics buffer"<<endl;
filename = path + "/statIndex";
MMapBuffer* indexBufferFile = MMapBuffer::create(filename.c_str(), 0);
char* indexBuffer = indexBufferFile->get_address();
string statFilename = path + "/subject_statis";
repo->subjectStat = OneConstantStatisticsBuffer::load(StatisticsBuffer::SUBJECT_STATIS, statFilename, indexBuffer);
statFilename = path + "/object_statis";
repo->objectStat = OneConstantStatisticsBuffer::load(StatisticsBuffer::OBJECT_STATIS, statFilename, indexBuffer);
statFilename = path + "/subjectpredicate_statis";
repo->subPredicateStat = TwoConstantStatisticsBuffer::load(StatisticsBuffer::SUBJECTPREDICATE_STATIS, statFilename, indexBuffer);
statFilename = path + "/objectpredicate_statis";
repo->objPredicateStat = TwoConstantStatisticsBuffer::load(StatisticsBuffer::OBJECTPREDICATE_STATIS, statFilename, indexBuffer);
#ifdef DEBUG
cout<<"subject count: "<<((OneConstantStatisticsBuffer*)repo->subjectStat)->getEntityCount()<<endl;
cout<<"object count: "<<((OneConstantStatisticsBuffer*)repo->objectStat)->getEntityCount()<<endl;
#endif
repo->buffer = new EntityIDBuffer();
repo->columnFinder = new FindEntityID(repo);
cerr<<"load complete!"<<endl;
repo->bitmapQuery = new TripleBitQuery(*repo);
repo->query = new RDFQuery(repo->bitmapQuery, repo);
return repo;
}
