DiskLoc MmapV1ExtentManager::_createExtent( OperationContext* txn,
int size,
bool enforceQuota ) {
size = quantizeExtentSize( size );
if ( size > maxSize() )
size = maxSize();
verify( size < DataFile::maxSize() );
for ( int i = numFiles() - 1; i >= 0; i-- ) {
DataFile* f = _getOpenFile(i);
invariant(f);
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( txn, i, f, size, enforceQuota );
}
}
_checkQuota( enforceQuota, numFiles() );
// no space in an existing file
// allocate files until we either get one big enough or hit maxSize
for ( int i = 0; i < 8; i++ ) {
DataFile* f = _addAFile( txn, size, false );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( txn, numFiles() - 1, f, size, enforceQuota );
}
}
// callers don't check for null return code, so assert
msgasserted(14810, "couldn't allocate space for a new extent" );
}
Extent* ExtentManager::createExtent(const char *ns, int size, bool newCapped, bool enforceQuota ) {
size = quantizeExtentSize( size );
for ( int i = numFiles() - 1; i >= 0; i-- ) {
DataFile* f = getFile( i );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( i, f, ns, size, newCapped, enforceQuota );
}
}
// no space in an existing file
// allocate files until we either get one big enough or hit maxSize
for ( int i = 0; i < 8; i++ ) {
DataFile* f = addAFile( size, false );
if ( f->getHeader()->unusedLength >= size ||
f->getHeader()->fileLength >= DataFile::maxSize() ) {
return _createExtentInFile( numFiles() - 1, f, ns, size, newCapped, enforceQuota );
}
}
// callers don't check for null return code, so assert
msgasserted(14810, "couldn't allocate space for a new extent" );
}
DataHeader *get_dataheader(const stdString &dir, const stdString &file,
FileOffset offset)
{
DataFile *datafile = DataFile::reference(dir, file, false);
if (!datafile)
return 0;
DataHeader *header = datafile->getHeader(offset);
datafile->release(); // ref'ed by header
return header; // might be NULL
}
// Either sets header to new dirname/basename/offset
// or returns false
DataHeader *OldDataReader::getHeader(const stdString &dirname,
const stdString &basename,
FileOffset offset)
{
if (!Filename::isValid(basename))
return 0;
DataFile *datafile =
DataFile::reference(dirname, basename, false);
DataHeader *new_header = datafile->getHeader(offset);
datafile->release(); // now ref'ed by header
return new_header;
}
DiskLoc MmapV1ExtentManager::_createExtent( TransactionExperiment* txn,
int size,
int maxFileNoForQuota ) {
size = quantizeExtentSize( size );
if ( size > maxSize() )
size = maxSize();
verify( size < DataFile::maxSize() );
for ( int i = numFiles() - 1; i >= 0; i-- ) {
DataFile* f = getFile( txn, i );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( txn, i, f, size, maxFileNoForQuota );
}
}
if ( maxFileNoForQuota > 0 &&
static_cast<int>( numFiles() ) >= maxFileNoForQuota &&
!cc().hasWrittenSinceCheckpoint() ) {
_quotaExceeded();
}
// no space in an existing file
// allocate files until we either get one big enough or hit maxSize
for ( int i = 0; i < 8; i++ ) {
DataFile* f = _addAFile( txn, size, false );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( txn, numFiles() - 1, f, size, maxFileNoForQuota );
}
}
// callers don't check for null return code, so assert
msgasserted(14810, "couldn't allocate space for a new extent" );
}
Extent* ExtentManager::createExtent(const char *ns, int size, bool newCapped, bool enforceQuota ) {
size = quantizeExtentSize( size );
if ( size > Extent::maxSize() )
size = Extent::maxSize();
verify( size < DataFile::maxSize() );
for ( int i = numFiles() - 1; i >= 0; i-- ) {
DataFile* f = getFile( i );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( i, f, ns, size, newCapped, enforceQuota );
}
}
if ( enforceQuota &&
fileIndexExceedsQuota( ns, numFiles() ) &&
!cc().hasWrittenThisPass() ) {
_quotaExceeded();
}
// no space in an existing file
// allocate files until we either get one big enough or hit maxSize
for ( int i = 0; i < 8; i++ ) {
DataFile* f = addAFile( size, false );
if ( f->getHeader()->unusedLength >= size ) {
return _createExtentInFile( numFiles() - 1, f, ns, size, newCapped, enforceQuota );
}
}
// callers don't check for null return code, so assert
msgasserted(14810, "couldn't allocate space for a new extent" );
}
void MMAPV1DatabaseCatalogEntry::markIndexSafe24AndUp( OperationContext* opCtx ) {
if ( _extentManager.numFiles() == 0 )
return;
int major = 0;
int minor = 0;
_extentManager.getFileFormat( opCtx, &major, &minor );
invariant( major == PDFILE_VERSION );
if ( minor == PDFILE_VERSION_MINOR_24_AND_NEWER )
return;
invariant( minor == PDFILE_VERSION_MINOR_22_AND_OLDER );
DataFile* df = _extentManager.getFile( opCtx, 0 );
opCtx->recoveryUnit()->writingInt(df->getHeader()->versionMinor) =
PDFILE_VERSION_MINOR_24_AND_NEWER;
}
void count_channel_values(RTree *tree, const stdString &directory,
DbrType &type, DbrCount &count, size_t &blocks, size_t &values)
{
type = 0;
count = 0;
blocks = 0;
values = 0;
if (! tree)
return;
DataFile *datafile;
AutoPtr<DataHeader> header;
RTree::Datablock block;
RTree::Node node(tree->getM(), true);
stdString start, end;
int idx;
bool ok;
for (ok = tree->getFirstDatablock(node, idx, block);
ok;
ok = tree->getNextDatablock(node, idx, block))
{
++blocks;
datafile = DataFile::reference(directory,
block.data_filename,
false);
header = datafile->getHeader(block.data_offset);
datafile->release();
if (header)
{
if (blocks == 1)
{
type = header->data.dbr_type;
count = header->data.dbr_count;
}
values += header->data.num_samples;
header = 0;
}
else
printf("Cannot read header in data file.\n");
}
}
// ran at startup.
static void repairDatabasesAndCheckVersion(bool shouldClearNonLocalTmpCollections) {
// LastError * le = lastError.get( true );
LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl;
Lock::GlobalWrite lk;
vector< string > dbNames;
getDatabaseNames( dbNames );
for ( vector< string >::iterator i = dbNames.begin(); i != dbNames.end(); ++i ) {
string dbName = *i;
LOG(1) << "\t" << dbName << endl;
Client::Context ctx( dbName );
DataFile *p = ctx.db()->getExtentManager().getFile( 0 );
DataFileHeader *h = p->getHeader();
if ( replSettings.usingReplSets() ) {
// we only care about the _id index if we are in a replset
checkForIdIndexes(ctx.db());
}
if (shouldClearNonLocalTmpCollections || dbName == "local")
ctx.db()->clearTmpCollections();
if (!h->isCurrentVersion() || mongodGlobalParams.repair) {
if( h->version <= 0 ) {
uasserted(14026,
str::stream() << "db " << dbName << " appears corrupt pdfile version: " << h->version
<< " info: " << h->versionMinor << ' ' << h->fileLength);
}
if ( !h->isCurrentVersion() ) {
log() << "****" << endl;
log() << "****" << endl;
log() << "need to upgrade database " << dbName << " "
<< "with pdfile version " << h->version << "." << h->versionMinor << ", "
<< "new version: "
<< PDFILE_VERSION << "." << PDFILE_VERSION_MINOR_22_AND_OLDER
<< endl;
}
if (mongodGlobalParams.upgrade) {
// QUESTION: Repair even if file format is higher version than code?
doDBUpgrade( dbName, h );
}
else {
log() << "\t Not upgrading, exiting" << endl;
log() << "\t run --upgrade to upgrade dbs, then start again" << endl;
log() << "****" << endl;
dbexit( EXIT_NEED_UPGRADE );
mongodGlobalParams.upgrade = 1;
return;
}
}
else {
const string systemIndexes = cc().database()->name() + ".system.indexes";
auto_ptr<Runner> runner(InternalPlanner::collectionScan(systemIndexes));
BSONObj index;
Runner::RunnerState state;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&index, NULL))) {
const BSONObj key = index.getObjectField("key");
const string plugin = IndexNames::findPluginName(key);
if (h->versionMinor == PDFILE_VERSION_MINOR_22_AND_OLDER) {
if (IndexNames::existedBefore24(plugin))
continue;
log() << "Index " << index << " claims to be of type '" << plugin << "', "
<< "which is either invalid or did not exist before v2.4. "
<< "See the upgrade section: "
<< "http://dochub.mongodb.org/core/upgrade-2.4"
<< startupWarningsLog;
}
const Status keyStatus = validateKeyPattern(key);
if (!keyStatus.isOK()) {
log() << "Problem with index " << index << ": " << keyStatus.reason()
<< " This index can still be used however it cannot be rebuilt."
<< " For more info see"
<< " http://dochub.mongodb.org/core/index-validation"
<< startupWarningsLog;
}
}
if (Runner::RUNNER_EOF != state) {
warning() << "Internal error while reading collection " << systemIndexes;
}
Database::closeDatabase(dbName.c_str(), storageGlobalParams.dbpath);
}
}
LOG(1) << "done repairDatabases" << endl;
if (mongodGlobalParams.upgrade) {
log() << "finished checking dbs" << endl;
cc().shutdown();
dbexit( EXIT_CLEAN );
}
}
DataWriter::DataWriter(Index &index,
const stdString &channel_name,
const CtrlInfo &ctrl_info,
DbrType dbr_type,
DbrCount dbr_count,
double period,
size_t num_samples)
: index(index),
channel_name(channel_name),
ctrl_info(ctrl_info),
dbr_type(dbr_type),
dbr_count(dbr_count),
period(period),
raw_value_size(RawValue::getSize(dbr_type, dbr_count)),
next_buffer_size(0),
available(0)
{
DataFile *datafile = 0;
try
{
// Size of next buffer should at least hold num_samples
calc_next_buffer_size(num_samples);
// Find or add appropriate data buffer
tree = index.addChannel(channel_name, directory);
RTree::Datablock block;
RTree::Node node(tree->getM(), true);
int idx;
if (tree->getLastDatablock(node, idx, block))
{ // - There is a data file and buffer
datafile = DataFile::reference(directory, block.data_filename, true);
header = datafile->getHeader(block.data_offset);
datafile->release(); // now ref'ed by header
datafile = 0;
// See if anything has changed
CtrlInfo prev_ctrl_info;
prev_ctrl_info.read(header->datafile,
header->data.ctrl_info_offset);
if (prev_ctrl_info != ctrl_info)
// Add new header and new ctrl_info
addNewHeader(true);
else if (header->data.dbr_type != dbr_type ||
header->data.dbr_count != dbr_count)
// Add new header, use existing ctrl_info
addNewHeader(false);
else
{ // All fine, just check if we're already in bigger league
size_t capacity = header->capacity();
if (capacity > num_samples)
calc_next_buffer_size(capacity);
}
}
else // New data file, add the initial header
addNewHeader(true);
available = header->available();
}
catch (GenericException &e)
{
tree = 0;
if (datafile)
datafile->release();
throw GenericException(__FILE__, __LINE__,
"Channel '%s':\n%s",
channel_name.c_str(),
e.what());
}
}
unsigned long dump_datablocks_for_channel(IndexFile &index,
const stdString &channel_name,
unsigned long &direct_count,
unsigned long &chained_count)
{
DataFile *datafile;
AutoPtr<DataHeader> header;
direct_count = chained_count = 0;
stdString directory;
AutoPtr<RTree> tree(index.getTree(channel_name, directory));
if (! tree)
return 0;
RTree::Datablock block;
RTree::Node node(tree->getM(), true);
stdString start, end;
int idx;
bool ok;
if (verbose > 1)
printf("RTree M for channel '%s': %d\n", channel_name.c_str(),
tree->getM());
if (verbose > 2)
printf("Datablocks for channel '%s':\n", channel_name.c_str());
for (ok = tree->getFirstDatablock(node, idx, block);
ok;
ok = tree->getNextDatablock(node, idx, block))
{
++direct_count;
if (verbose > 2)
printf("'%s' @ 0x%lX: Indexed range %s - %s\n",
block.data_filename.c_str(),
(unsigned long)block.data_offset,
epicsTimeTxt(node.record[idx].start, start),
epicsTimeTxt(node.record[idx].end, end));
if (verbose > 3)
{
datafile = DataFile::reference(directory,
block.data_filename,
false);
header = datafile->getHeader(block.data_offset);
datafile->release();
if (header)
{
header->show(stdout, true);
header = 0;
}
else
printf("Cannot read header in data file.\n");
}
bool first_hidden_block = true;
while (tree->getNextChainedBlock(block))
{
if (first_hidden_block && verbose > 2)
{
first_hidden_block = false;
printf("Hidden blocks with smaller time range:\n");
}
++chained_count;
if (verbose > 2)
{
printf("--- '%s' @ 0x%lX\n",
block.data_filename.c_str(),
(unsigned long)block.data_offset);
if (verbose > 3)
{
datafile = DataFile::reference(directory,
block.data_filename,
false);
header = datafile->getHeader(block.data_offset);
if (header)
{
header->show(stdout, false);
header = 0;
}
else
printf("Cannot read header in data file.\n");
datafile->release();
}
}
}
printf("\n");
}
return direct_count + chained_count;
}
void convert_dir_index(int RTreeM,
const stdString &dir_name, const stdString &index_name)
{
OldDirectoryFile dir;
if (!dir.open(dir_name))
return;
OldDirectoryFileIterator channels = dir.findFirst();
IndexFile index(RTreeM);
stdString index_directory;
if (verbose)
printf("Opened directory file '%s'\n", dir_name.c_str());
index.open(index_name, false);
if (verbose)
printf("Created index '%s'\n", index_name.c_str());
for (/**/; channels.isValid(); channels.next())
{
if (verbose)
printf("Channel '%s':\n", channels.entry.data.name);
if (! Filename::isValid(channels.entry.data.first_file))
{
if (verbose)
printf("No values\n");
continue;
}
AutoPtr<RTree> tree(index.addChannel(channels.entry.data.name, index_directory));
if (!tree)
{
fprintf(stderr, "Cannot add channel '%s' to index '%s'\n",
channels.entry.data.name, index_name.c_str());
continue;
}
DataFile *datafile =
DataFile::reference(dir.getDirname(),
channels.entry.data.first_file, false);
AutoPtr<DataHeader> header(
datafile->getHeader(channels.entry.data.first_offset));
datafile->release();
while (header && header->isValid())
{
if (verbose)
{
stdString start, end;
epicsTime2string(header->data.begin_time, start);
epicsTime2string(header->data.end_time, end);
printf("'%s' @ 0x0%lX: %s - %s\n",
header->datafile->getBasename().c_str(),
(unsigned long)header->offset,
start.c_str(),
end.c_str());
}
if (!tree->insertDatablock(
header->data.begin_time, header->data.end_time,
header->offset, header->datafile->getBasename()))
{
fprintf(stderr, "insertDatablock failed for channel '%s'\n",
channels.entry.data.name);
break;
}
header->read_next();
}
}
DataFile::close_all();
}
