void BitmapBuffer::save() { //loadfromtemp(); static bool first = true; string filename = dir + "/BitmapBuffer"; MMapBuffer * buffer; //new MMapBuffer(filename.c_str(), 0); string predicateFile(filename); predicateFile.append("_predicate"); MMapBuffer * predicateBuffer =new MMapBuffer(predicateFile.c_str(), predicate_managers[0].size() * (sizeof(ID) + sizeof(size_t)) * 2); char* predicateWriter = predicateBuffer->get_address(); char* bufferWriter = NULL; map<ID, ChunkManager*>::const_iterator iter = predicate_managers[0].begin(); char* startPtr; size_t offset = 0; startPtr = iter->second->ptrs[0]; if(first == true) { buffer = new MMapBuffer(filename.c_str(), iter->second->meta->length[0]); //offset = buffer->get_offset(); //first = false; } else { //buffer = new MMapBuffer(filename.c_str(), iter->second->meta->length[0], true); //offset = buffer->get_offset(); } predicateWriter = predicateBuffer->get_address(); bufferWriter = buffer->get_address(); vector<size_t>::iterator pageNoIter = iter->second->usedPage[0].begin(), limit = iter->second->usedPage[0].end(); for(; pageNoIter != limit; pageNoIter++ ) { size_t pageNo = *pageNoIter; memcpy(bufferWriter, temp1->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); bufferWriter = bufferWriter + MemoryBuffer::pagesize; } *((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID); *((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t); offset = offset + iter->second->meta->length[0]; bufferWriter = buffer->resize(iter->second->meta->length[1]); char* startPos = bufferWriter + offset; pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end(); for(; pageNoIter != limit; pageNoIter++ ) { size_t pageNo = *pageNoIter; memcpy(startPos, temp2->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); startPos = startPos + MemoryBuffer::pagesize; } assert(iter->second->meta->length[1] == iter->second->usedPage[1].size() * MemoryBuffer::pagesize); offset = offset + iter->second->meta->length[1]; iter++; for(; iter != predicate_managers[0].end(); iter++) { bufferWriter = buffer->resize(iter->second->meta->length[0]); startPos = bufferWriter + offset; pageNoIter = iter->second->usedPage[0].begin(); limit = iter->second->usedPage[0].end(); for(; pageNoIter != limit; pageNoIter++) { size_t pageNo = *pageNoIter; memcpy(startPos, temp1->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); startPos = startPos + MemoryBuffer::pagesize; } //cout<<"used page count: "<<iter->second->usedPage[0].size()<<endl; //iter->second->meta->endPtr[0] = startPos + iter->second->meta->usedSpace[0]; //used to build index; *((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID); *((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t); offset += iter->second->meta->length[0]; assert(iter->second->usedPage[0].size() * MemoryBuffer::pagesize == iter->second->meta->length[0]); bufferWriter = buffer->resize(iter->second->meta->length[1]); startPos = bufferWriter + offset; //iter->second->meta->startPtr[1] = startPos; //used to build index; //iter->second->meta->endPtr[1] = startPos + iter->second->meta->usedSpace[1]; pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end(); for(; pageNoIter != limit; pageNoIter++) { size_t pageNo = *pageNoIter; memcpy(startPos, temp2->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); startPos = startPos + MemoryBuffer::pagesize; } offset += iter->second->meta->length[1]; assert(iter->second->usedPage[1].size() * MemoryBuffer::pagesize == iter->second->meta->length[1]); } buffer->flush(); temp1->discard(); temp2->discard(); iter = predicate_managers[1].begin(); for(; iter != predicate_managers[1].end(); iter++) { bufferWriter = buffer->resize(iter->second->meta->length[0]); startPos = bufferWriter + offset; pageNoIter = iter->second->usedPage[0].begin(); limit = iter->second->usedPage[0].end(); for(; pageNoIter != limit; pageNoIter++) { size_t pageNo = *pageNoIter; memcpy(startPos, temp3->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); startPos = startPos + MemoryBuffer::pagesize; } *((ID*)predicateWriter) = iter->first; predicateWriter = predicateWriter + sizeof(ID); *((size_t*)predicateWriter) = offset; predicateWriter = predicateWriter + sizeof(size_t); offset += iter->second->meta->length[0]; assert(iter->second->usedPage[0].size() * MemoryBuffer::pagesize == iter->second->meta->length[0]); bufferWriter = buffer->resize(iter->second->usedPage[1].size() * MemoryBuffer::pagesize); startPos = bufferWriter + offset; pageNoIter = iter->second->usedPage[1].begin(); limit = iter->second->usedPage[1].end(); for(; pageNoIter != limit; pageNoIter++) { size_t pageNo = *pageNoIter; memcpy(startPos, temp4->get_address() + pageNo * MemoryBuffer::pagesize, MemoryBuffer::pagesize); startPos = startPos + MemoryBuffer::pagesize; } offset += iter->second->meta->length[1]; assert(iter->second->usedPage[1].size() * MemoryBuffer::pagesize == iter->second->meta->length[1]); } buffer->flush(); predicateBuffer->flush(); predicateWriter = predicateBuffer->get_address(); int i = 0; ID id; for(iter = predicate_managers[0].begin(); iter != predicate_managers[0].end(); iter++, i++) { id = *((ID*)predicateWriter); assert(iter->first == id); predicateWriter = predicateWriter + sizeof(ID); offset = *((size_t*)predicateWriter); predicateWriter = predicateWriter + sizeof(size_t); char* base = buffer->get_address() + offset; iter->second->meta = (ChunkManagerMeta*)base; iter->second->meta->startPtr[0] = base + sizeof(ChunkManagerMeta); iter->second->meta->endPtr[0] = iter->second->meta->startPtr[0] + iter->second->meta->usedSpace[0]; iter->second->meta->startPtr[1] = base + iter->second->meta->length[0]; iter->second->meta->endPtr[1] = iter->second->meta->startPtr[1] + iter->second->meta->usedSpace[1]; //::printMeta(*(iter->second->meta)); } for(iter = predicate_managers[1].begin(); iter != predicate_managers[1].end(); iter++, i++) { id = *((ID*)predicateWriter); assert(iter->first == id); predicateWriter = predicateWriter + sizeof(ID); offset = *((size_t*)predicateWriter); predicateWriter = predicateWriter + sizeof(size_t); char* base = buffer->get_address() + offset; iter->second->meta = (ChunkManagerMeta*)base; iter->second->meta->startPtr[0] = base + sizeof(ChunkManagerMeta); iter->second->meta->endPtr[0] = iter->second->meta->startPtr[0] + iter->second->meta->usedSpace[0]; iter->second->meta->startPtr[1] = base + iter->second->meta->length[0]; iter->second->meta->endPtr[1] = iter->second->meta->startPtr[1] + iter->second->meta->usedSpace[1]; //::printMeta(*(iter->second->meta)); } temp3->discard(); temp4->discard(); //build index; MMapBuffer* bitmapIndex = NULL; #ifdef DEBUG cout<<"build hash index for subject"<<endl; #endif for ( map<ID,ChunkManager*>::iterator iter = predicate_managers[0].begin(); iter != predicate_managers[0].end(); iter++ ) { if ( iter->second != NULL ) { #ifdef DEBUG cout<<iter->first<<endl; #endif iter->second->buildChunkIndex(); iter->second->getChunkIndex(1)->save(bitmapIndex); iter->second->getChunkIndex(2)->save(bitmapIndex); } } #ifdef DEBUG cout<<"build hash index for object"<<endl; #endif for ( map<ID, ChunkManager*>::iterator iter = predicate_managers[1].begin(); iter != predicate_managers[1].end(); iter++ ) { if ( iter->second != NULL ) { #ifdef DEBUF cout<<iter->first<<endl; #endif iter->second->buildChunkIndex(); iter->second->getChunkIndex(1)->save(bitmapIndex); iter->second->getChunkIndex(2)->save(bitmapIndex); } } delete bitmapIndex; delete buffer; delete predicateBuffer; }
TripleBitRepository* TripleBitRepository::create(const string path) { TripleBitRepository* repo = new TripleBitRepository(); string filename = path + "/BitmapBuffer"; if(OSFile::fileExists(filename) == false) { //file dose not exist, repository has not been build; fprintf(stderr, "database files dose not exist!"); return NULL; } // load the repository from image files; //load bitmap #ifdef DEBUG cout<<filename.c_str()<<endl; #endif repo->bitmapImage = new MMapBuffer(filename.c_str(), 0); string predicateFile(filename); predicateFile.append("_predicate"); string indexFile(filename); indexFile.append("_index"); repo->bitmapPredicateImage = new MMapBuffer(predicateFile.c_str(), 0); repo->bitmapIndexImage = new MMapBuffer(indexFile.c_str(), 0); repo->bitmapBuffer = BitmapBuffer::load(repo->bitmapImage, repo->bitmapIndexImage, repo->bitmapPredicateImage); repo->UriTable = URITable::load(path); repo->preTable = PredicateTable::load(path); #ifdef DEBUG cout<<"total triple count: "<<repo->bitmapBuffer->getTripleCount()<<endl; cout<<"URITableSize: "<<repo->UriTable->getSize()<<endl; cout<<"predicateTableSize: "<<repo->preTable->getSize()<<endl; #endif // cout<<"begin load statistics buffer"<<endl; filename = path + "/statIndex"; MMapBuffer* indexBufferFile = MMapBuffer::create(filename.c_str(), 0); char* indexBuffer = indexBufferFile->get_address(); string statFilename = path + "/subject_statis"; repo->subjectStat = OneConstantStatisticsBuffer::load(StatisticsBuffer::SUBJECT_STATIS, statFilename, indexBuffer); statFilename = path + "/object_statis"; repo->objectStat = OneConstantStatisticsBuffer::load(StatisticsBuffer::OBJECT_STATIS, statFilename, indexBuffer); statFilename = path + "/subjectpredicate_statis"; repo->subPredicateStat = TwoConstantStatisticsBuffer::load(StatisticsBuffer::SUBJECTPREDICATE_STATIS, statFilename, indexBuffer); statFilename = path + "/objectpredicate_statis"; repo->objPredicateStat = TwoConstantStatisticsBuffer::load(StatisticsBuffer::OBJECTPREDICATE_STATIS, statFilename, indexBuffer); #ifdef DEBUG cout<<"subject count: "<<((OneConstantStatisticsBuffer*)repo->subjectStat)->getEntityCount()<<endl; cout<<"object count: "<<((OneConstantStatisticsBuffer*)repo->objectStat)->getEntityCount()<<endl; #endif repo->buffer = new EntityIDBuffer(); repo->columnFinder = new FindEntityID(repo); cerr<<"load complete!"<<endl; repo->bitmapQuery = new TripleBitQuery(*repo); repo->query = new RDFQuery(repo->bitmapQuery, repo); return repo; }