__declspec(noinline) void makeChunkWritableOld(size_t chunkno) { scoped_lock lk(mapViewMutex); if( writable.get(chunkno) ) return; size_t loc = chunkno * MemoryMappedFile::ChunkSize; void *Loc = (void*) loc; size_t ofs; MongoMMF *mmf = privateViews.find( (void *) (loc), ofs ); MemoryMappedFile *f = (MemoryMappedFile*) mmf; assert(f); size_t len = MemoryMappedFile::ChunkSize; assert( mmf->getView() <= Loc ); if( ofs + len > f->length() ) { // at the very end of the map len = (size_t) (f->length() - ofs); } else { ; } // todo: check this goes away on remap DWORD old; bool ok = VirtualProtect(Loc, len, PAGE_WRITECOPY, &old); if( !ok ) { DWORD e = GetLastError(); log() << "VirtualProtect failed " << Loc << ' ' << len << ' ' << errnoWithDescription(e) << endl; assert(false); } writable.set(chunkno); }
//--------------------------------------------------------------------------- void dumpFacts(ofstream& out,TempFile& rawFacts,const string& name) // Dump the facts { // Sort the facts TempFile sortedFacts(rawFacts.getBaseFile()); Sorter::sort(rawFacts,sortedFacts,skipTriple,compareTriple,true); // Dump the facts { unlink("facts.sql"); ofstream out("facts.sql"); MemoryMappedFile in; in.open(sortedFacts.getFile().c_str()); const Triple* triplesBegin=reinterpret_cast<const Triple*>(in.getBegin()); const Triple* triplesEnd=reinterpret_cast<const Triple*>(in.getEnd()); for (const Triple* iter=triplesBegin,*limit=triplesEnd; iter!=limit; ++iter) out << (*iter).subject << "\t" << (*iter).predicate << "\t" << (*iter).object << std::endl; } // And write the copy statement out << "drop schema if exists " << name << " cascade;" << endl; out << "create schema " << name << ";" << endl; out << "create table " << name << ".facts(subject int not null, predicate int not null, object int not null);" << endl; out << "copy " << name << ".facts from 'facts.sql';" << endl; // Create indices out << "create index facts_spo on " << name << ".facts (subject, predicate, object);" << endl; out << "create index facts_pso on " << name << ".facts (predicate, subject, object);" << endl; out << "create index facts_pos on " << name << ".facts (predicate, object, subject);" << endl; }
/** remember "last sequence number" to speed recoveries */ void lsnThread() { Client::initThread("lsn"); time_t last = 0; while( 1 ) { unsigned long long lsn = j.toStoreLastSeqNum.take(); // if you are on a really fast fsync interval, we don't write this as often if( time(0) - last < 5 ) continue; last = time(0); try { // os can flush as it likes. if it flushes slowly, we will just do extra work on recovery. // however, given we actually close the file, that seems unlikely. MemoryMappedFile f; unsigned long long length = 8; unsigned long long *L = static_cast<unsigned long long*>(f.map(lsnPath().string().c_str(), length)); assert(L); *L = lsn; } catch(std::exception& e) { log() << "write to lsn file fails " << e.what() << endl; } } }
void processDiagLog( const char * file ) { Connection c; MemoryMappedFile f; long length; unsigned long long L = 0; char * root = (char*)f.map( file , L, MemoryMappedFile::SEQUENTIAL ); verify( L < 0x80000000 ); length = (long) L; verify( root ); verify( length > 0 ); char * pos = root; long read = 0; while ( read < length ) { Message m(pos,false); int len = m.header()->len; DbMessage d(m); cout << len << " " << d.getns() << endl; processMessage( c , m ); read += len; pos += len; } f.close(); }
bool InstructionContext::TranslateEvmFile(LPCTSTR szInFile, LPCTSTR szOutFile) { MemoryMappedFile mmf; if (!mmf.MapFlie(szInFile)) { cerr << "Cannot open evm file.\n"; return false; } Header* header = (Header*) mmf.data(); trace("Header codeSize = %u, dataSize = %u, initialDataSize = %u\n", header->codeSize, header->dataSize, header->initialDataSize); // verify if (header->signature != SIGNATURE_ESET_VM1 || header->dataSize < header->initialDataSize || header->codeSize * 3 + header->initialDataSize + 20 != mmf.size()) { cerr << "Invalid evm file.\n"; return false; } Instruction* instruction = (Instruction*)(header + 1); //section.push_back(0xCC); // Debug vector<size_t> instructionOffsets; instructionOffsets.resize(header->codeSize); for (uint32_t i = 0; i < header->codeSize; ++i, ++instruction) { instructionOffsets[i] = section.size(); if (!AddInstruction( instruction->Opcode, instruction->Destination, instruction->Source, i, header->dataSize)) { return false; } } // Handle jumps for (auto it(jumps.begin()), itEnd(jumps.end()); it != itEnd; ++it) { for each(auto offset in it->second) { uint32_t* pos = (uint32_t*) §ion[offset]; size_t instructionIndex = it->first + 1; if (instructionIndex >= instructionOffsets.size()) { cerr << "Error: Jump address out of bounds\n"; return false; } *pos = instructionOffsets[instructionIndex] - sizeof(uint32_t) - offset; } } return MakeExecutable(szOutFile, instruction, header->initialDataSize, header->dataSize); }
unsigned long long journalReadLSN() { try { // os can flush as it likes. if it flushes slowly, we will just do extra work on recovery. // however, given we actually close the file, that seems unlikely. MemoryMappedFile f; unsigned long long *L = static_cast<unsigned long long*>(f.map(lsnPath().string().c_str())); assert(L); return *L; } catch(std::exception& e) { log() << "couldn't read journal/lsn file - if a recovery is needed will apply all files. " << e.what() << endl; } return 0; }
//--------------------------------------------------------------------------- int main(int argc,char* argv[]) { // Check the arguments if (argc<2) { cerr << "usage: " << argv[0] << " <tmp file> " << endl; return 1; } // Parse the input { ofstream out("yago_aftertest.txt"); MemoryMappedFile in; const char* iter,*limit; ensure(in.open(argv[1])); iter=in.getBegin(); limit=in.getEnd(); while (iter != limit){ uint64_t i1, i2, i3; iter = TempFile::readId(iter, i1); iter = TempFile::readId(iter, i2); iter = TempFile::readId(iter, i3); unsigned node = i1; iter = TempFile::readId(iter, i1); iter = TempFile::readId(iter, i2); iter = TempFile::readId(iter, i3); unsigned dir = i2; iter = TempFile::readId(iter, i1); iter = TempFile::readId(iter, i2); iter = TempFile::readId(iter, i3); unsigned selectivity = i3; iter = TempFile::readId(iter, i1); iter = TempFile::readId(iter, i2); iter = TempFile::readId(iter, i3); out<<node<<" "<<dir<<" "<<selectivity<<endl; } } char a = 1000000; cerr<<static_cast<unsigned>(a)<<endl; cout << "Done." << endl; }
impl(const char *path, int width, int height, int bit_count) : m_writable{ true } { size_t file_size = 0; if (width < 0 || height < 0) throw std::invalid_argument{ "invalid bitmap dimensions" }; if (bit_count != 24 && bit_count != 32) throw std::invalid_argument{ "unsupported biBitCount" }; file_size += sizeof(BITMAPFILEHEADER); file_size += sizeof(BITMAPINFOHEADER); file_size += height * bitmap_row_size(width, bit_count); m_mmap = MemoryMappedFile{ path, file_size, MemoryMappedFile::CREATE_TAG }; m_bitmap = BitmapFileData{ m_mmap.size(), m_mmap.write_ptr(), true }; m_bitmap.init(m_mmap.size(), width, height, bit_count); }
int test2_old9() { out() << "test2" << endl; printStackTrace(); if ( 1 ) return 1; MemoryMappedFile f; long len = 64*1024*1024; char *p = (char *) f.map("/tmp/test.dat", len); char *start = p; char *end = p + 64*1024*1024-2; end[1] = 'z'; int i; while ( p < end ) { *p++ = ' '; if ( ++i%64 == 0 ) { *p++ = '\n'; *p++ = 'x'; } } *p = 'a'; f.flush(true); out() << "done" << endl; char *x = start + 32 * 1024 * 1024; char *y = start + 48 * 1024 * 1024; char *z = start + 62 * 1024 * 1024; strcpy(z, "zfoo"); out() << "y" << endl; strcpy(y, "yfoo"); strcpy(x, "xfoo"); strcpy(start, "xfoo"); dbexit( EXIT_TEST ); return 1; }
void drillDown( path root ) { if ( is_directory( root ) ) { directory_iterator end; directory_iterator i(root); while ( i != end ) { path p = *i; drillDown( p ); i++; } return; } if ( ! ( endsWith( root.string().c_str() , ".bson" ) || endsWith( root.string().c_str() , ".bin" ) ) ) { cerr << "don't know what to do with [" << root.string() << "]" << endl; return; } out() << root.string() << endl; string ns; { string dir = root.branch_path().string(); if ( dir.find( "/" ) == string::npos ) ns += dir; else ns += dir.substr( dir.find_last_of( "/" ) + 1 ); } { string l = root.leaf(); l = l.substr( 0 , l.find_last_of( "." ) ); ns += "." + l; } if ( boost::filesystem::file_size( root ) == 0 ) { out() << "file " + root.native_file_string() + " empty, aborting" << endl; return; } out() << "\t going into namespace [" << ns << "]" << endl; MemoryMappedFile mmf; assert( mmf.map( root.string().c_str() ) ); char * data = (char*)mmf.viewOfs(); int read = 0; int num = 0; while ( read < mmf.length() ) { BSONObj o( data ); conn().insert( ns.c_str() , o ); read += o.objsize(); data += o.objsize(); if ( ! ( ++num % 1000 ) ) out() << "read " << read << "/" << mmf.length() << " bytes so far. " << num << " objects" << endl; } out() << "\t " << num << " objects" << endl; }
Status TripleBitBuilder::resolveTriples(TempFile& rawFacts, TempFile& facts) { cerr<<"Sort by Subject"<<endl; ID subjectID, objectID, predicateID; ID lastSubject = 0, lastObject = 0, lastPredicate = 0; unsigned count0 = 0, count1 = 0; TempFile sortedBySubject("./SortByS"), sortedByObject("./SortByO"); Sorter::sort(rawFacts, sortedBySubject, skipIdIdId, compare123); { //insert into chunk sortedBySubject.close(); MemoryMappedFile mappedIn; assert(mappedIn.open(sortedBySubject.getFile().c_str())); const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd(); loadTriple(reader, subjectID, predicateID, objectID); lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID; reader = skipIdIdId(reader); bool v = generateXY(subjectID, objectID); bitmap->insertTriple(predicateID, subjectID, objectID, v, 0); count0 = count1 = 1; while (reader < limit) { loadTriple(reader, subjectID, predicateID, objectID); if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) { reader = skipIdIdId(reader); continue; } if ( subjectID != lastSubject ) { ((OneConstantStatisticsBuffer*)statBuffer[0])->addStatis(lastSubject, count0); statBuffer[2]->addStatis(lastSubject, lastPredicate, count1); lastPredicate = predicateID; lastSubject = subjectID; lastObject = objectID; count0 = count1 = 1; } else if ( predicateID != lastPredicate ) { statBuffer[2]->addStatis(lastSubject, lastPredicate, count1); lastPredicate = predicateID; lastObject = objectID; count0++; count1 = 1; }else { count0++; count1++; lastObject = objectID; } reader = reader + 12; v = generateXY(subjectID, objectID); //0 indicate the triple is sorted by subjects' id; bitmap->insertTriple(predicateID, subjectID, objectID, v, 0); } mappedIn.close(); } bitmap->flush(); ((OneConstantStatisticsBuffer*)statBuffer[0])->flush(); ((TwoConstantStatisticsBuffer*)statBuffer[2])->flush(); //sort cerr << "Sort by Object" << endl; Sorter::sort(rawFacts, sortedByObject, skipIdIdId, compare321); { //insert into chunk sortedByObject.close(); MemoryMappedFile mappedIn; assert(mappedIn.open(sortedByObject.getFile().c_str())); const char* reader = mappedIn.getBegin(), *limit = mappedIn.getEnd(); loadTriple(reader, subjectID, predicateID, objectID); lastSubject = subjectID; lastPredicate = predicateID; lastObject = objectID; reader = skipIdIdId(reader); bool v = generateXY(objectID, subjectID); bitmap->insertTriple(predicateID, objectID, subjectID, v, 1); count0 = count1 = 1; while (reader < limit) { loadTriple(reader, subjectID, predicateID, objectID); if(lastSubject == subjectID && lastPredicate == predicateID && lastObject == objectID) { reader = skipIdIdId(reader); continue; } if ( objectID != lastObject ) { ((OneConstantStatisticsBuffer*)statBuffer[1])->addStatis(lastObject, count0); statBuffer[3]->addStatis(lastObject, lastPredicate, count1); lastPredicate = predicateID; lastObject = objectID; lastSubject = subjectID; count0 = count1 = 1; } else if ( predicateID != lastPredicate ) { statBuffer[3]->addStatis(lastObject, lastPredicate, count1); lastPredicate = predicateID; lastSubject = subjectID; count0++; count1 = 1; } else { lastSubject = subjectID; count0++; count1++; } reader = skipIdIdId(reader); v = generateXY(objectID, subjectID); // 1 indicate the triple is sorted by objects' id; bitmap->insertTriple(predicateID, objectID, subjectID, v, 1); } mappedIn.close(); } bitmap->flush(); ((OneConstantStatisticsBuffer*)statBuffer[1])->flush(); ((TwoConstantStatisticsBuffer*)statBuffer[3])->flush(); rawFacts.discard(); sortedByObject.discard(); sortedBySubject.discard(); return OK; }
void close() { m_mmap.close(); }
void flush() { m_mmap.flush(); }
/* if you want recording of the timings, place the password for the perf database in ./../settings.py: pstatspassword="******" */ void connect() { if( once ) return; ++once; // no writing to perf db if _DEBUG DEV return; const char *fn = "../../settings.py"; if( !exists(fn) ) { if( exists("settings.py") ) fn = "settings.py"; else { cout << "no ../../settings.py or ./settings.py file found. will not write perf stats to pstats db." << endl; cout << "it is recommended this be enabled even on dev boxes" << endl; return; } } try { if( conn == 0 ) { MemoryMappedFile f; const char *p = (const char *) f.mapWithOptions(fn, MongoFile::READONLY); string pwd; { const char *q = str::after(p, "pstatspassword=\""); if( *q == 0 ) { cout << "info perftests.cpp: no pstatspassword= in settings.py" << endl; return; } else { pwd = str::before(q, '\"'); } } boost::shared_ptr<DBClientConnection> c(new DBClientConnection(false, 0, 60)); string err; if( c->connect("perfdb.10gen.cc", err) ) { if( !c->auth("perf", "perf", pwd, err) ) { cout << "info: authentication with stats db failed: " << err << endl; assert(false); } conn = c; // override the hostname with the buildbot hostname, if present ifstream hostf( "../../info/host" ); if ( hostf.good() ) { char buf[1024]; hostf.getline(buf, sizeof(buf)); _perfhostname = buf; } else { _perfhostname = getHostName(); } } else { cout << err << " (to log perfstats)" << endl; } } } catch(...) { } }
//--------------------------------------------------------------------------- static void buildDictionary(TempFile& rawStrings,TempFile& stringTable,TempFile& stringIds,map<unsigned,unsigned>& subTypes) // Build the dictionary { cerr << "Building the dictionary..." << endl; // Sort the strings to resolve duplicates TempFile sortedStrings(rawStrings.getBaseFile()); Sorter::sort(rawStrings,sortedStrings,skipStringIdId,compareStringIdId); rawStrings.discard(); // Build the id map and the string list TempFile rawIdMap(rawStrings.getBaseFile()),stringList(rawStrings.getBaseFile()); { MemoryMappedFile strings; ensure(strings.open(sortedStrings.getFile().c_str())); uint64_t lastId=0; unsigned lastLen=0; const char* lastStr=0; uint64_t lastType=0; for (const char* iter=strings.getBegin(),*limit=strings.getEnd();iter!=limit;) { // Read the entry unsigned stringLen; const char* stringStart; iter=TempFile::readString(iter,stringLen,stringStart); uint64_t id,type; iter=TempFile::readId(iter,type); iter=TempFile::readId(iter,id); // A new one? if ((!lastStr)||(stringLen!=lastLen)||(memcmp(lastStr,stringStart,stringLen)!=0)||(type!=lastType)) { stringList.writeId(id); stringList.writeString(stringLen,stringStart); stringList.writeId(type); rawIdMap.writeId(id); rawIdMap.writeId(id); lastId=id; lastLen=stringLen; lastStr=stringStart; lastType=type; } else { rawIdMap.writeId(lastId); rawIdMap.writeId(id); } } } sortedStrings.discard(); // Sort the string list Sorter::sort(stringList,stringTable,skipIdStringId,compareId); stringList.discard(); // Sort the ID map TempFile idMap(rawStrings.getBaseFile()); Sorter::sort(rawIdMap,idMap,skipIdId,compareId); rawIdMap.discard(); // Construct new ids TempFile newIds(rawStrings.getBaseFile()); { MemoryMappedFile in; ensure(in.open(idMap.getFile().c_str())); uint64_t lastId=0,newId=0; for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) { uint64_t firstId,currentId; iter=TempFile::readId(iter,firstId); iter=TempFile::readId(iter,currentId); if (firstId!=lastId) { ++newId; lastId=firstId; } newIds.writeId(currentId); newIds.writeId(newId); if (subTypes.count(currentId)) subTypes[currentId]=newId; } } // And a final sort Sorter::sort(newIds,stringIds,skipIdId,compareValue); newIds.discard(); // Resolve the subtypes if necessary if (!subTypes.empty()) { TempFile fixedTypes(rawStrings.getBaseFile()); MemoryMappedFile in; ensure(in.open(stringTable.getFile().c_str())); for (const char* iter=in.getBegin(),*limit=in.getEnd();iter!=limit;) { uint64_t id,typeInfo; const char* value; unsigned valueLen; iter=TempFile::readId(TempFile::readString(TempFile::readId(iter,id),valueLen,value),typeInfo); unsigned type=typeInfo&0xFF,subType=(typeInfo>>8); if (Type::hasSubType(static_cast<Type::ID>(type))) { assert(subTypes.count(subType)); typeInfo=type|(subTypes[subType]<<8); } else { assert(subType==0); } fixedTypes.writeId(id); fixedTypes.writeString(valueLen,value); fixedTypes.writeId(typeInfo); } fixedTypes.close(); fixedTypes.swap(stringTable); }
void run() { string fn = "/tmp/testfile.map"; boost::filesystem::remove(fn); MemoryMappedFile f; char *p = (char *) f.create(fn, 1024 * 1024 * 1024, true); verify(p); strcpy(p, "hello"); { void *x = f.testGetCopyOnWriteView(); Timer tt; for( int i = 11; i < 1000000000; i++ ) p[i] = 'z'; cout << "fill 1GB time: " << tt.millis() << "ms" << endl; f.testCloseCopyOnWriteView(x); } /* test a lot of view/unviews */ { Timer t; char *q; for( int i = 0; i < 1000; i++ ) { q = (char *) f.testGetCopyOnWriteView(); verify( q ); if( i == 999 ) { strcpy(q+2, "there"); } f.testCloseCopyOnWriteView(q); } cout << "view unview: " << t.millis() << "ms" << endl; } f.flush(true); /* plain old mmaped writes */ { Timer t; for( int i = 0; i < 10; i++ ) { memset(p+100, 'c', 200 * 1024 * 1024); } cout << "traditional writes: " << t.millis() << "ms" << endl; } f.flush(true); /* test doing some writes */ { Timer t; char *q = (char *) f.testGetCopyOnWriteView(); for( int i = 0; i < 10; i++ ) { verify( q ); memset(q+100, 'c', 200 * 1024 * 1024); } f.testCloseCopyOnWriteView(q); cout << "inc style some writes: " << t.millis() << "ms" << endl; } /* test doing some writes */ { Timer t; for( int i = 0; i < 10; i++ ) { char *q = (char *) f.testGetCopyOnWriteView(); verify( q ); memset(q+100, 'c', 200 * 1024 * 1024); f.testCloseCopyOnWriteView(q); } cout << "some writes: " << t.millis() << "ms" << endl; } /* more granular */ { Timer t; for( int i = 0; i < 100; i++ ) { char *q = (char *) f.testGetCopyOnWriteView(); verify( q ); memset(q+100, 'c', 20 * 1024 * 1024); f.testCloseCopyOnWriteView(q); } cout << "more granular some writes: " << t.millis() << "ms" << endl; } p[10] = 0; cout << p << endl; }
void say(unsigned long long n, int ms, string s) { unsigned long long rps = n*1000/ms; cout << "stats " << setw(33) << left << s << ' ' << right << setw(9) << rps << ' ' << right << setw(5) << ms << "ms "; if( showDurStats() ) cout << dur::stats.curr->_asCSV(); cout << endl; /* if you want recording of the timings, place the password for the perf database in ./../settings.py: pstatspassword="******" */ const char *fn = "../../settings.py"; static bool ok = true; if( ok ) { DEV { // no writing to perf db if dev } else if( !exists(fn) ) { cout << "no ../../settings.py file found. will not write perf stats to db" << endl; } else { try { if( conn == 0 ) { MemoryMappedFile f; const char *p = (const char *) f.mapWithOptions(fn, MongoFile::READONLY); string pwd; { const char *q = str::after(p, "pstatspassword=\""); if( *q == 0 ) { cout << "info perftests.cpp: no pstatspassword= in settings.py" << endl; ok = false; } else { pwd = str::before(q, '\"'); } } if( ok ) { conn = new DBClientConnection(false, 0, 10); string err; if( conn->connect("mongo05.10gen.cust.cbici.net", err) ) { if( !conn->auth("perf", "perf", pwd, err) ) { cout << "info: authentication with stats db failed: " << err << endl; assert(false); } } else { cout << err << " (to log perfstats)" << endl; ok = false; } } } if( conn && !conn->isFailed() ) { const char *ns = "perf.pstats"; if( perfHist ) { static bool needver = true; try { // try to report rps from last time */ Query q; { BSONObjBuilder b; b.append("host",getHostName()).append("test",s).append("dur",cmdLine.dur); DEV b.append("info.DEBUG",true); else b.appendNull("info.DEBUG"); if( sizeof(int*) == 4 ) b.append("info.bits", 32); else b.appendNull("info.bits"); q = Query(b.obj()).sort("when",-1); } //cout << q.toString() << endl; BSONObj fields = BSON( "rps" << 1 << "info" << 1 ); vector<BSONObj> v; conn->findN(v, ns, q, perfHist, 0, &fields); for( vector<BSONObj>::iterator i = v.begin(); i != v.end(); i++ ) { BSONObj o = *i; double lastrps = o["rps"].Number(); if( lastrps ) { cout << "stats " << setw(33) << right << "new/old:" << ' ' << setw(9); cout << fixed << setprecision(2) << rps / lastrps; if( needver ) { cout << " " << o.getFieldDotted("info.git").toString(); } cout << '\n'; } } } catch(...) { } cout.flush(); needver = false; } { bob b; b.append("host", getHostName()); b.appendTimeT("when", time(0)); b.append("test", s); b.append("rps", (int) rps); b.append("millis", ms); b.appendBool("dur", cmdLine.dur); if( showDurStats() && cmdLine.dur ) b.append("durStats", dur::stats.curr->_asObj()); { bob inf; inf.append("version", versionString); if( sizeof(int*) == 4 ) inf.append("bits", 32); DEV inf.append("DEBUG", true); #if defined(_WIN32) inf.append("os", "win"); #endif inf.append("git", gitVersion()); inf.append("boost", BOOST_VERSION); b.append("info", inf.obj()); } BSONObj o = b.obj(); //cout << "inserting " << o.toString() << endl; conn->insert(ns, o); } }