DiskLoc MmapV1ExtentManager::_createExtent( OperationContext* txn, int size, bool enforceQuota ) { size = quantizeExtentSize( size ); if ( size > maxSize() ) size = maxSize(); verify( size < DataFile::maxSize() ); for ( int i = numFiles() - 1; i >= 0; i-- ) { DataFile* f = _getOpenFile(i); invariant(f); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( txn, i, f, size, enforceQuota ); } } _checkQuota( enforceQuota, numFiles() ); // no space in an existing file // allocate files until we either get one big enough or hit maxSize for ( int i = 0; i < 8; i++ ) { DataFile* f = _addAFile( txn, size, false ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( txn, numFiles() - 1, f, size, enforceQuota ); } } // callers don't check for null return code, so assert msgasserted(14810, "couldn't allocate space for a new extent" ); }
Extent* ExtentManager::createExtent(const char *ns, int size, bool newCapped, bool enforceQuota ) { size = quantizeExtentSize( size ); for ( int i = numFiles() - 1; i >= 0; i-- ) { DataFile* f = getFile( i ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( i, f, ns, size, newCapped, enforceQuota ); } } // no space in an existing file // allocate files until we either get one big enough or hit maxSize for ( int i = 0; i < 8; i++ ) { DataFile* f = addAFile( size, false ); if ( f->getHeader()->unusedLength >= size || f->getHeader()->fileLength >= DataFile::maxSize() ) { return _createExtentInFile( numFiles() - 1, f, ns, size, newCapped, enforceQuota ); } } // callers don't check for null return code, so assert msgasserted(14810, "couldn't allocate space for a new extent" ); }
DataHeader *get_dataheader(const stdString &dir, const stdString &file, FileOffset offset) { DataFile *datafile = DataFile::reference(dir, file, false); if (!datafile) return 0; DataHeader *header = datafile->getHeader(offset); datafile->release(); // ref'ed by header return header; // might be NULL }
// Either sets header to new dirname/basename/offset // or returns false DataHeader *OldDataReader::getHeader(const stdString &dirname, const stdString &basename, FileOffset offset) { if (!Filename::isValid(basename)) return 0; DataFile *datafile = DataFile::reference(dirname, basename, false); DataHeader *new_header = datafile->getHeader(offset); datafile->release(); // now ref'ed by header return new_header; }
DiskLoc MmapV1ExtentManager::_createExtent( TransactionExperiment* txn, int size, int maxFileNoForQuota ) { size = quantizeExtentSize( size ); if ( size > maxSize() ) size = maxSize(); verify( size < DataFile::maxSize() ); for ( int i = numFiles() - 1; i >= 0; i-- ) { DataFile* f = getFile( txn, i ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( txn, i, f, size, maxFileNoForQuota ); } } if ( maxFileNoForQuota > 0 && static_cast<int>( numFiles() ) >= maxFileNoForQuota && !cc().hasWrittenSinceCheckpoint() ) { _quotaExceeded(); } // no space in an existing file // allocate files until we either get one big enough or hit maxSize for ( int i = 0; i < 8; i++ ) { DataFile* f = _addAFile( txn, size, false ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( txn, numFiles() - 1, f, size, maxFileNoForQuota ); } } // callers don't check for null return code, so assert msgasserted(14810, "couldn't allocate space for a new extent" ); }
Extent* ExtentManager::createExtent(const char *ns, int size, bool newCapped, bool enforceQuota ) { size = quantizeExtentSize( size ); if ( size > Extent::maxSize() ) size = Extent::maxSize(); verify( size < DataFile::maxSize() ); for ( int i = numFiles() - 1; i >= 0; i-- ) { DataFile* f = getFile( i ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( i, f, ns, size, newCapped, enforceQuota ); } } if ( enforceQuota && fileIndexExceedsQuota( ns, numFiles() ) && !cc().hasWrittenThisPass() ) { _quotaExceeded(); } // no space in an existing file // allocate files until we either get one big enough or hit maxSize for ( int i = 0; i < 8; i++ ) { DataFile* f = addAFile( size, false ); if ( f->getHeader()->unusedLength >= size ) { return _createExtentInFile( numFiles() - 1, f, ns, size, newCapped, enforceQuota ); } } // callers don't check for null return code, so assert msgasserted(14810, "couldn't allocate space for a new extent" ); }
void MMAPV1DatabaseCatalogEntry::markIndexSafe24AndUp( OperationContext* opCtx ) { if ( _extentManager.numFiles() == 0 ) return; int major = 0; int minor = 0; _extentManager.getFileFormat( opCtx, &major, &minor ); invariant( major == PDFILE_VERSION ); if ( minor == PDFILE_VERSION_MINOR_24_AND_NEWER ) return; invariant( minor == PDFILE_VERSION_MINOR_22_AND_OLDER ); DataFile* df = _extentManager.getFile( opCtx, 0 ); opCtx->recoveryUnit()->writingInt(df->getHeader()->versionMinor) = PDFILE_VERSION_MINOR_24_AND_NEWER; }
void count_channel_values(RTree *tree, const stdString &directory, DbrType &type, DbrCount &count, size_t &blocks, size_t &values) { type = 0; count = 0; blocks = 0; values = 0; if (! tree) return; DataFile *datafile; AutoPtr<DataHeader> header; RTree::Datablock block; RTree::Node node(tree->getM(), true); stdString start, end; int idx; bool ok; for (ok = tree->getFirstDatablock(node, idx, block); ok; ok = tree->getNextDatablock(node, idx, block)) { ++blocks; datafile = DataFile::reference(directory, block.data_filename, false); header = datafile->getHeader(block.data_offset); datafile->release(); if (header) { if (blocks == 1) { type = header->data.dbr_type; count = header->data.dbr_count; } values += header->data.num_samples; header = 0; } else printf("Cannot read header in data file.\n"); } }
// ran at startup. static void repairDatabasesAndCheckVersion(bool shouldClearNonLocalTmpCollections) { // LastError * le = lastError.get( true ); LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl; Lock::GlobalWrite lk; vector< string > dbNames; getDatabaseNames( dbNames ); for ( vector< string >::iterator i = dbNames.begin(); i != dbNames.end(); ++i ) { string dbName = *i; LOG(1) << "\t" << dbName << endl; Client::Context ctx( dbName ); DataFile *p = ctx.db()->getExtentManager().getFile( 0 ); DataFileHeader *h = p->getHeader(); if ( replSettings.usingReplSets() ) { // we only care about the _id index if we are in a replset checkForIdIndexes(ctx.db()); } if (shouldClearNonLocalTmpCollections || dbName == "local") ctx.db()->clearTmpCollections(); if (!h->isCurrentVersion() || mongodGlobalParams.repair) { if( h->version <= 0 ) { uasserted(14026, str::stream() << "db " << dbName << " appears corrupt pdfile version: " << h->version << " info: " << h->versionMinor << ' ' << h->fileLength); } if ( !h->isCurrentVersion() ) { log() << "****" << endl; log() << "****" << endl; log() << "need to upgrade database " << dbName << " " << "with pdfile version " << h->version << "." << h->versionMinor << ", " << "new version: " << PDFILE_VERSION << "." << PDFILE_VERSION_MINOR_22_AND_OLDER << endl; } if (mongodGlobalParams.upgrade) { // QUESTION: Repair even if file format is higher version than code? doDBUpgrade( dbName, h ); } else { log() << "\t Not upgrading, exiting" << endl; log() << "\t run --upgrade to upgrade dbs, then start again" << endl; log() << "****" << endl; dbexit( EXIT_NEED_UPGRADE ); mongodGlobalParams.upgrade = 1; return; } } else { const string systemIndexes = cc().database()->name() + ".system.indexes"; auto_ptr<Runner> runner(InternalPlanner::collectionScan(systemIndexes)); BSONObj index; Runner::RunnerState state; while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&index, NULL))) { const BSONObj key = index.getObjectField("key"); const string plugin = IndexNames::findPluginName(key); if (h->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER) { if (IndexNames::existedBefore24(plugin)) continue; log() << "Index " << index << " claims to be of type '" << plugin << "', " << "which is either invalid or did not exist before v2.4. " << "See the upgrade section: " << "http://dochub.mongodb.org/core/upgrade-2.4" << startupWarningsLog; } const Status keyStatus = validateKeyPattern(key); if (!keyStatus.isOK()) { log() << "Problem with index " << index << ": " << keyStatus.reason() << " This index can still be used however it cannot be rebuilt." << " For more info see" << " http://dochub.mongodb.org/core/index-validation" << startupWarningsLog; } } if (Runner::RUNNER_EOF != state) { warning() << "Internal error while reading collection " << systemIndexes; } Database::closeDatabase(dbName.c_str(), storageGlobalParams.dbpath); } } LOG(1) << "done repairDatabases" << endl; if (mongodGlobalParams.upgrade) { log() << "finished checking dbs" << endl; cc().shutdown(); dbexit( EXIT_CLEAN ); } }
DataWriter::DataWriter(Index &index, const stdString &channel_name, const CtrlInfo &ctrl_info, DbrType dbr_type, DbrCount dbr_count, double period, size_t num_samples) : index(index), channel_name(channel_name), ctrl_info(ctrl_info), dbr_type(dbr_type), dbr_count(dbr_count), period(period), raw_value_size(RawValue::getSize(dbr_type, dbr_count)), next_buffer_size(0), available(0) { DataFile *datafile = 0; try { // Size of next buffer should at least hold num_samples calc_next_buffer_size(num_samples); // Find or add appropriate data buffer tree = index.addChannel(channel_name, directory); RTree::Datablock block; RTree::Node node(tree->getM(), true); int idx; if (tree->getLastDatablock(node, idx, block)) { // - There is a data file and buffer datafile = DataFile::reference(directory, block.data_filename, true); header = datafile->getHeader(block.data_offset); datafile->release(); // now ref'ed by header datafile = 0; // See if anything has changed CtrlInfo prev_ctrl_info; prev_ctrl_info.read(header->datafile, header->data.ctrl_info_offset); if (prev_ctrl_info != ctrl_info) // Add new header and new ctrl_info addNewHeader(true); else if (header->data.dbr_type != dbr_type || header->data.dbr_count != dbr_count) // Add new header, use existing ctrl_info addNewHeader(false); else { // All fine, just check if we're already in bigger league size_t capacity = header->capacity(); if (capacity > num_samples) calc_next_buffer_size(capacity); } } else // New data file, add the initial header addNewHeader(true); available = header->available(); } catch (GenericException &e) { tree = 0; if (datafile) datafile->release(); throw GenericException(__FILE__, __LINE__, "Channel '%s':\n%s", channel_name.c_str(), e.what()); } }
unsigned long dump_datablocks_for_channel(IndexFile &index, const stdString &channel_name, unsigned long &direct_count, unsigned long &chained_count) { DataFile *datafile; AutoPtr<DataHeader> header; direct_count = chained_count = 0; stdString directory; AutoPtr<RTree> tree(index.getTree(channel_name, directory)); if (! tree) return 0; RTree::Datablock block; RTree::Node node(tree->getM(), true); stdString start, end; int idx; bool ok; if (verbose > 1) printf("RTree M for channel '%s': %d\n", channel_name.c_str(), tree->getM()); if (verbose > 2) printf("Datablocks for channel '%s':\n", channel_name.c_str()); for (ok = tree->getFirstDatablock(node, idx, block); ok; ok = tree->getNextDatablock(node, idx, block)) { ++direct_count; if (verbose > 2) printf("'%s' @ 0x%lX: Indexed range %s - %s\n", block.data_filename.c_str(), (unsigned long)block.data_offset, epicsTimeTxt(node.record[idx].start, start), epicsTimeTxt(node.record[idx].end, end)); if (verbose > 3) { datafile = DataFile::reference(directory, block.data_filename, false); header = datafile->getHeader(block.data_offset); datafile->release(); if (header) { header->show(stdout, true); header = 0; } else printf("Cannot read header in data file.\n"); } bool first_hidden_block = true; while (tree->getNextChainedBlock(block)) { if (first_hidden_block && verbose > 2) { first_hidden_block = false; printf("Hidden blocks with smaller time range:\n"); } ++chained_count; if (verbose > 2) { printf("--- '%s' @ 0x%lX\n", block.data_filename.c_str(), (unsigned long)block.data_offset); if (verbose > 3) { datafile = DataFile::reference(directory, block.data_filename, false); header = datafile->getHeader(block.data_offset); if (header) { header->show(stdout, false); header = 0; } else printf("Cannot read header in data file.\n"); datafile->release(); } } } printf("\n"); } return direct_count + chained_count; }
void convert_dir_index(int RTreeM, const stdString &dir_name, const stdString &index_name) { OldDirectoryFile dir; if (!dir.open(dir_name)) return; OldDirectoryFileIterator channels = dir.findFirst(); IndexFile index(RTreeM); stdString index_directory; if (verbose) printf("Opened directory file '%s'\n", dir_name.c_str()); index.open(index_name, false); if (verbose) printf("Created index '%s'\n", index_name.c_str()); for (/**/; channels.isValid(); channels.next()) { if (verbose) printf("Channel '%s':\n", channels.entry.data.name); if (! Filename::isValid(channels.entry.data.first_file)) { if (verbose) printf("No values\n"); continue; } AutoPtr<RTree> tree(index.addChannel(channels.entry.data.name, index_directory)); if (!tree) { fprintf(stderr, "Cannot add channel '%s' to index '%s'\n", channels.entry.data.name, index_name.c_str()); continue; } DataFile *datafile = DataFile::reference(dir.getDirname(), channels.entry.data.first_file, false); AutoPtr<DataHeader> header( datafile->getHeader(channels.entry.data.first_offset)); datafile->release(); while (header && header->isValid()) { if (verbose) { stdString start, end; epicsTime2string(header->data.begin_time, start); epicsTime2string(header->data.end_time, end); printf("'%s' @ 0x0%lX: %s - %s\n", header->datafile->getBasename().c_str(), (unsigned long)header->offset, start.c_str(), end.c_str()); } if (!tree->insertDatablock( header->data.begin_time, header->data.end_time, header->offset, header->datafile->getBasename())) { fprintf(stderr, "insertDatablock failed for channel '%s'\n", channels.entry.data.name); break; } header->read_next(); } } DataFile::close_all(); }