IndexAccessMethod* MMAPV1DatabaseCatalogEntry::getIndex( OperationContext* txn,
const CollectionCatalogEntry* collection,
IndexCatalogEntry* entry ) {
const string& type = entry->descriptor()->getAccessMethodName();
string ns = collection->ns().ns();
if ( IndexNames::TEXT == type ||
entry->descriptor()->getInfoElement("expireAfterSeconds").isNumber() ) {
NamespaceDetailsRSV1MetaData md( ns,
_namespaceIndex.details( ns ),
_getNamespaceRecordStore( txn, ns ) );
md.setUserFlag( txn, NamespaceDetails::Flag_UsePowerOf2Sizes );
}
RecordStore* rs = _getRecordStore( txn, entry->descriptor()->indexNamespace() );
invariant( rs );
if (IndexNames::HASHED == type)
return new HashAccessMethod( entry, rs );
if (IndexNames::GEO_2DSPHERE == type)
return new S2AccessMethod( entry, rs );
if (IndexNames::TEXT == type)
return new FTSAccessMethod( entry, rs );
if (IndexNames::GEO_HAYSTACK == type)
return new HaystackAccessMethod( entry, rs );
if ("" == type)
return new BtreeAccessMethod( entry, rs );
if (IndexNames::GEO_2D == type)
return new TwoDAccessMethod( entry, rs );
log() << "Can't find index for keyPattern " << entry->descriptor()->keyPattern();
fassertFailed(17489);
}
~RepairFileDeleter() {
if ( _success )
return;
log() << "cleaning up failed repair "
<< "db: " << _dbName << " path: " << _pathString;
try {
getDur().syncDataAndTruncateJournal();
MongoFile::flushAll(true); // need both in case journaling is disabled
{
Client::Context tempContext( _dbName, _pathString );
Database::closeDatabase( _dbName, _pathString );
}
MONGO_ASSERT_ON_EXCEPTION( boost::filesystem::remove_all( _path ) );
}
catch ( DBException& e ) {
error() << "RepairFileDeleter failed to cleanup: " << e;
error() << "aborting";
fassertFailed( 17402 );
}
}
void* MemoryMappedFile::createPrivateMap() {
verify( maphandle );
scoped_lock lk(mapViewMutex);
void* privateMapAddress = MapViewOfFile(
maphandle, // file mapping handle
FILE_MAP_READ, // access
0, 0, // file offset, high and low
0 ); // bytes to map, 0 == all
if ( privateMapAddress == 0 ) {
DWORD dosError = GetLastError();
log() << "MapViewOfFile for " << filename()
<< " failed with error " << errnoWithDescription( dosError )
<< " (file size is " << len << ")"
<< " in MemoryMappedFile::createPrivateMap"
<< endl;
fassertFailed( 16167 );
}
clearWritableBits( privateMapAddress );
views.push_back( privateMapAddress );
memconcept::is( privateMapAddress, memconcept::concept::memorymappedfile, filename() );
return privateMapAddress;
}
void* MemoryMappedFile::createReadOnlyMap() {
verify( maphandle );
scoped_lock lk(mapViewMutex);
LPVOID thisAddress = getNextMemoryMappedFileLocation( len );
void* readOnlyMapAddress = MapViewOfFileEx(
maphandle, // file mapping handle
FILE_MAP_READ, // access
0, 0, // file offset, high and low
0, // bytes to map, 0 == all
thisAddress ); // address to place file
if ( 0 == readOnlyMapAddress ) {
DWORD dosError = GetLastError();
log() << "MapViewOfFileEx for " << filename()
<< " failed with error " << errnoWithDescription( dosError )
<< " (file size is " << len << ")"
<< " in MemoryMappedFile::createReadOnlyMap"
<< endl;
fassertFailed( 16165 );
}
memconcept::is( readOnlyMapAddress, memconcept::concept::other, filename() );
views.push_back( readOnlyMapAddress );
return readOnlyMapAddress;
}
void WriteBatchExecutor::execRemove( const BatchItemRef& removeItem,
WriteErrorDetail** error ) {
// Removes are similar to updates, but page faults are handled externally
// BEGIN CURRENT OP
scoped_ptr<CurOp> currentOp( beginCurrentOp( _client, removeItem ) );
incOpStats( removeItem );
WriteOpResult result;
// NOTE: Deletes will not fault outside the lock once any data has been written
{
PageFaultRetryableSection pageFaultSection;
while ( true ) {
try {
multiRemove( removeItem, &result );
break;
}
catch (PageFaultException& pfe) {
pfe.touch();
invariant(!result.getError());
continue;
}
fassertFailed(17429);
}
}
// END CURRENT OP
incWriteStats( removeItem, result.getStats(), result.getError(), currentOp.get() );
finishCurrentOp( _client, currentOp.get(), result.getError() );
if ( result.getError() ) {
result.getError()->setIndex( removeItem.getItemIndex() );
*error = result.releaseError();
}
}
bool ComparisonMatchExpression::matchesSingleElement( const BSONElement& e ) const {
//log() << "\t ComparisonMatchExpression e: " << e << " _rhs: " << _rhs << "\n"
//<< toString() << std::endl;
if ( e.canonicalType() != _rhs.canonicalType() ) {
// some special cases
// jstNULL and undefined are treated the same
if ( e.canonicalType() + _rhs.canonicalType() == 5 ) {
return matchType() == EQ || matchType() == LTE || matchType() == GTE;
}
if ( _rhs.type() == MaxKey || _rhs.type() == MinKey ) {
return matchType() != EQ;
}
return false;
}
int x = compareElementValues( e, _rhs );
//log() << "\t\t" << x << endl;
switch ( matchType() ) {
case LT:
return x < 0;
case LTE:
return x <= 0;
case EQ:
return x == 0;
case GT:
return x > 0;
case GTE:
return x >= 0;
default:
fassertFailed( 16828 );
}
}
void BackgroundSync::producerThread() {
Client::initThread("rsBackgroundSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
_threadPoolTaskExecutor.startup();
ON_BLOCK_EXIT([this]() {
_threadPoolTaskExecutor.shutdown();
_threadPoolTaskExecutor.join();
});
while (!inShutdown()) {
try {
_producerThread();
} catch (const DBException& e) {
std::string msg(str::stream() << "sync producer problem: " << e.toString());
error() << msg;
_replCoord->setMyHeartbeatMessage(msg);
} catch (const std::exception& e2) {
severe() << "sync producer exception: " << e2.what();
fassertFailed(28546);
}
}
stop();
}
void SSLManager::_handleSSLError(int code) {
switch (code) {
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
// should not happen because we turned on AUTO_RETRY
error() << "SSL error: " << code << endl;
fassertFailed( 16676 );
break;
case SSL_ERROR_SYSCALL:
if (code < 0) {
error() << "socket error: " << errnoWithDescription() << endl;
throw SocketException(SocketException::CONNECT_ERROR, "");
}
error() << "could not negotiate SSL connection: EOF detected" << endl;
throw SocketException(SocketException::CONNECT_ERROR, "");
break;
case SSL_ERROR_SSL:
{
int ret = ERR_get_error();
error() << _getSSLErrorMessage(ret) << endl;
throw SocketException(SocketException::CONNECT_ERROR, "");
break;
}
case SSL_ERROR_ZERO_RETURN:
error() << "could not negotiate SSL connection: EOF detected" << endl;
throw SocketException(SocketException::CONNECT_ERROR, "");
break;
default:
error() << "unrecognized SSL error" << endl;
throw SocketException(SocketException::CONNECT_ERROR, "");
break;
}
}
Status TransportLayerASIO::start() {
stdx::lock_guard<stdx::mutex> lk(_mutex);
_running.store(true);
// If we're in async mode then the ServiceExecutor will handle calling run_one() in a pool
// of threads. Otherwise we need a thread to just handle the async_accept calls.
if (!_listenerOptions.async) {
_listenerThread = stdx::thread([this] {
setThreadName("listener");
while (_running.load()) {
try {
_ioContext->run();
_ioContext->reset();
} catch (...) {
severe() << "Uncaught exception in the listener: " << exceptionToStatus();
fassertFailed(40491);
}
}
});
}
for (auto& acceptor : _acceptors) {
acceptor.listen();
_acceptConnection(acceptor);
}
const char* ssl = "";
#ifdef MONGO_CONFIG_SSL
if (_sslMode != SSLParams::SSLMode_disabled) {
ssl = " ssl";
}
#endif
log() << "waiting for connections on port " << _listenerOptions.port << ssl;
return Status::OK();
}
// Applies a batch of oplog entries, by using a set of threads to apply the operations and then
// writes the oplog entries to the local oplog.
OpTime SyncTail::multiApply(OperationContext* txn, const OpQueue& ops) {
invariant(_applyFunc);
if (getGlobalServiceContext()->getGlobalStorageEngine()->isMmapV1()) {
// Use a ThreadPool to prefetch all the operations in a batch.
prefetchOps(ops.getDeque(), &_prefetcherPool);
}
std::vector<std::vector<BSONObj>> writerVectors(replWriterThreadCount);
fillWriterVectors(ops.getDeque(), &writerVectors);
LOG(2) << "replication batch size is " << ops.getDeque().size() << endl;
// We must grab this because we're going to grab write locks later.
// We hold this mutex the entire time we're writing; it doesn't matter
// because all readers are blocked anyway.
stdx::lock_guard<SimpleMutex> fsynclk(filesLockedFsync);
// stop all readers until we're done
Lock::ParallelBatchWriterMode pbwm(txn->lockState());
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getMemberState().primary() && !replCoord->isWaitingForApplierToDrain()) {
severe() << "attempting to replicate ops while primary";
fassertFailed(28527);
}
applyOps(writerVectors, &_writerPool, _applyFunc, this);
OpTime lastOpTime = writeOpsToOplog(txn, ops.getDeque());
if (inShutdown()) {
return OpTime();
}
_writerPool.join();
// We have now written all database writes and updated the oplog to match.
return lastOpTime;
}
Status AuthzManagerExternalStateMongos::dropCollection(
const NamespaceString& collectionName) {
fassertFailed(17106);
}
Status AuthzManagerExternalStateMongos::remove(
const NamespaceString& collectionName,
const BSONObj& query) {
fassertFailed(17104);
}
Status AuthzManagerExternalStateMongos::createIndex(
const NamespaceString& collectionName,
const BSONObj& pattern,
bool unique) {
fassertFailed(17105);
}
Status AuthzManagerExternalStateMongos::findOne(
const NamespaceString& collectionName,
const BSONObj& query,
BSONObj* result) {
fassertFailed(17101);
}
Status AuthzManagerExternalStateMongos::insert(
const NamespaceString& collectionName,
const BSONObj& document) {
fassertFailed(17102);
}
bool AuthzManagerExternalStateMongos::tryLockUpgradeProcess() {
fassertFailed(17109);
}
Status AuthzManagerExternalStateMongod::renameCollection(
const NamespaceString& oldName,
const NamespaceString& newName,
const BSONObj& writeConcern) {
fassertFailed(17097);
}
void ReplicationCoordinatorImpl::_startElectSelf_inlock() {
invariant(!_freshnessChecker);
invariant(!_electCmdRunner);
switch (_rsConfigState) {
case kConfigSteady:
break;
case kConfigInitiating:
case kConfigReconfiguring:
case kConfigHBReconfiguring:
LOG(2) << "Not standing for election; processing a configuration change";
// Transition out of candidate role.
_topCoord->processLoseElection();
return;
default:
severe() << "Entered replica set election code while in illegal config state "
<< int(_rsConfigState);
fassertFailed(18913);
}
log() << "Standing for election";
const StatusWith<executor::TaskExecutor::EventHandle> finishEvh = _replExecutor->makeEvent();
if (finishEvh.getStatus() == ErrorCodes::ShutdownInProgress) {
return;
}
fassert(18680, finishEvh.getStatus());
_electionFinishedEvent = finishEvh.getValue();
LoseElectionGuard lossGuard(_topCoord.get(),
_replExecutor.get(),
&_freshnessChecker,
&_electCmdRunner,
&_electionFinishedEvent);
invariant(_rsConfig.getMemberAt(_selfIndex).isElectable());
OpTime lastOpTimeApplied(_getMyLastAppliedOpTime_inlock());
if (lastOpTimeApplied.isNull()) {
log() << "not trying to elect self, "
"do not yet have a complete set of data from any point in time"
" -- lastAppliedOpTime is null";
return;
}
_freshnessChecker.reset(new FreshnessChecker);
StatusWith<executor::TaskExecutor::EventHandle> nextPhaseEvh =
_freshnessChecker->start(_replExecutor.get(),
lastOpTimeApplied.getTimestamp(),
_rsConfig,
_selfIndex,
_topCoord->getMaybeUpHostAndPorts());
if (nextPhaseEvh.getStatus() == ErrorCodes::ShutdownInProgress) {
return;
}
fassert(18681, nextPhaseEvh.getStatus());
_replExecutor
->onEvent(nextPhaseEvh.getValue(),
stdx::bind(&ReplicationCoordinatorImpl::_onFreshnessCheckComplete, this))
.status_with_transitional_ignore();
lossGuard.dismiss();
}
void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(
TopologyCoordinator::StartElectionReason reason) {
invariant(!_voteRequester);
invariant(!_freshnessChecker);
switch (_rsConfigState) {
case kConfigSteady:
break;
case kConfigInitiating:
case kConfigReconfiguring:
case kConfigHBReconfiguring:
LOG(2) << "Not standing for election; processing a configuration change";
// Transition out of candidate role.
_topCoord->processLoseElection();
return;
default:
severe() << "Entered replica set election code while in illegal config state "
<< int(_rsConfigState);
fassertFailed(28641);
}
auto finishedEvent = _makeEvent();
if (!finishedEvent) {
return;
}
_electionFinishedEvent = finishedEvent;
auto dryRunFinishedEvent = _makeEvent();
if (!dryRunFinishedEvent) {
return;
}
_electionDryRunFinishedEvent = dryRunFinishedEvent;
LoseElectionDryRunGuardV1 lossGuard(this);
invariant(_rsConfig.getMemberAt(_selfIndex).isElectable());
const auto lastOpTime = _getMyLastAppliedOpTime_inlock();
if (lastOpTime == OpTime()) {
log() << "not trying to elect self, "
"do not yet have a complete set of data from any point in time";
return;
}
long long term = _topCoord->getTerm();
int primaryIndex = -1;
log() << "conducting a dry run election to see if we could be elected. current term: " << term;
_voteRequester.reset(new VoteRequester);
// Only set primaryIndex if the primary's vote is required during the dry run.
if (reason == TopologyCoordinator::StartElectionReason::kCatchupTakeover) {
primaryIndex = _topCoord->getCurrentPrimaryIndex();
}
StatusWith<executor::TaskExecutor::EventHandle> nextPhaseEvh =
_voteRequester->start(_replExecutor.get(),
_rsConfig,
_selfIndex,
term,
true, // dry run
lastOpTime,
primaryIndex);
if (nextPhaseEvh.getStatus() == ErrorCodes::ShutdownInProgress) {
return;
}
fassert(28685, nextPhaseEvh.getStatus());
_replExecutor
->onEvent(nextPhaseEvh.getValue(),
[=](const executor::TaskExecutor::CallbackArgs&) { _onDryRunComplete(term); })
.status_with_transitional_ignore();
lossGuard.dismiss();
}
bool ComparisonMatchExpression::matchesSingleElement(const BSONElement& e) const {
// log() << "\t ComparisonMatchExpression e: " << e << " _rhs: " << _rhs << "\n"
//<< toString() << std::endl;
if (e.canonicalType() != _rhs.canonicalType()) {
// some special cases
// jstNULL and undefined are treated the same
if (e.canonicalType() + _rhs.canonicalType() == 5) {
return matchType() == EQ || matchType() == LTE || matchType() == GTE;
}
if (_rhs.type() == MaxKey || _rhs.type() == MinKey) {
return matchType() != EQ;
}
return false;
}
// Special case handling for NaN. NaN is equal to NaN but
// otherwise always compares to false.
if (std::isnan(e.numberDouble()) || std::isnan(_rhs.numberDouble())) {
bool bothNaN = std::isnan(e.numberDouble()) && std::isnan(_rhs.numberDouble());
switch (matchType()) {
case LT:
return false;
case LTE:
return bothNaN;
case EQ:
return bothNaN;
case GT:
return false;
case GTE:
return bothNaN;
default:
// This is a comparison match expression, so it must be either
// a $lt, $lte, $gt, $gte, or equality expression.
fassertFailed(17448);
}
}
int x = compareElementValues(e, _rhs);
// log() << "\t\t" << x << endl;
switch (matchType()) {
case LT:
return x < 0;
case LTE:
return x <= 0;
case EQ:
return x == 0;
case GT:
return x > 0;
case GTE:
return x >= 0;
default:
// This is a comparison match expression, so it must be either
// a $lt, $lte, $gt, $gte, or equality expression.
fassertFailed(16828);
}
}
Status NetworkInterfaceASIO::startCommand(const TaskExecutor::CallbackHandle& cbHandle,
RemoteCommandRequest& request,
const RemoteCommandCompletionFn& onFinish) {
MONGO_ASIO_INVARIANT(onFinish, "Invalid completion function");
{
stdx::lock_guard<stdx::mutex> lk(_inProgressMutex);
const auto insertResult = _inGetConnection.emplace(cbHandle);
// We should never see the same CallbackHandle added twice
MONGO_ASIO_INVARIANT_INLOCK(insertResult.second, "Same CallbackHandle added twice");
}
if (inShutdown()) {
return {ErrorCodes::ShutdownInProgress, "NetworkInterfaceASIO shutdown in progress"};
}
LOG(2) << "startCommand: " << redact(request.toString());
auto getConnectionStartTime = now();
auto statusMetadata = attachMetadataIfNeeded(request, _metadataHook.get());
if (!statusMetadata.isOK()) {
return statusMetadata;
}
auto nextStep = [this, getConnectionStartTime, cbHandle, request, onFinish](
StatusWith<ConnectionPool::ConnectionHandle> swConn) {
if (!swConn.isOK()) {
LOG(2) << "Failed to get connection from pool for request " << request.id << ": "
<< swConn.getStatus();
bool wasPreviouslyCanceled = false;
{
stdx::lock_guard<stdx::mutex> lk(_inProgressMutex);
wasPreviouslyCanceled = _inGetConnection.erase(cbHandle) == 0;
}
Status status = wasPreviouslyCanceled
? Status(ErrorCodes::CallbackCanceled, "Callback canceled")
: swConn.getStatus();
if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) {
status = Status(ErrorCodes::ExceededTimeLimit, status.reason());
}
if (status.code() == ErrorCodes::ExceededTimeLimit) {
_numTimedOutOps.fetchAndAdd(1);
}
if (status.code() != ErrorCodes::CallbackCanceled) {
_numFailedOps.fetchAndAdd(1);
}
onFinish({status, now() - getConnectionStartTime});
signalWorkAvailable();
return;
}
auto conn = static_cast<connection_pool_asio::ASIOConnection*>(swConn.getValue().get());
AsyncOp* op = nullptr;
stdx::unique_lock<stdx::mutex> lk(_inProgressMutex);
const auto eraseCount = _inGetConnection.erase(cbHandle);
// If we didn't find the request, we've been canceled
if (eraseCount == 0) {
lk.unlock();
onFinish({ErrorCodes::CallbackCanceled,
"Callback canceled",
now() - getConnectionStartTime});
// Though we were canceled, we know that the stream is fine, so indicate success.
conn->indicateSuccess();
signalWorkAvailable();
return;
}
// We can't release the AsyncOp until we know we were not canceled.
auto ownedOp = conn->releaseAsyncOp();
op = ownedOp.get();
// This AsyncOp may be recycled. We expect timeout and canceled to be clean.
// If this op was most recently used to connect, its state transitions won't have been
// reset, so we do that here.
MONGO_ASIO_INVARIANT_INLOCK(!op->canceled(), "AsyncOp has dirty canceled flag", op);
MONGO_ASIO_INVARIANT_INLOCK(!op->timedOut(), "AsyncOp has dirty timeout flag", op);
op->clearStateTransitions();
// Now that we're inProgress, an external cancel can touch our op, but
// not until we release the inProgressMutex.
_inProgress.emplace(op, std::move(ownedOp));
op->_cbHandle = std::move(cbHandle);
op->_request = std::move(request);
op->_onFinish = std::move(onFinish);
op->_connectionPoolHandle = std::move(swConn.getValue());
op->startProgress(getConnectionStartTime);
// This ditches the lock and gets us onto the strand (so we're
// threadsafe)
op->_strand.post([this, op, getConnectionStartTime] {
const auto timeout = op->_request.timeout;
// Set timeout now that we have the correct request object
if (timeout != RemoteCommandRequest::kNoTimeout) {
// Subtract the time it took to get the connection from the pool from the request
// timeout.
auto getConnectionDuration = now() - getConnectionStartTime;
if (getConnectionDuration >= timeout) {
// We only assume that the request timer is guaranteed to fire *after* the
// timeout duration - but make no stronger assumption. It is thus possible that
// we have already exceeded the timeout. In this case we timeout the operation
// manually.
std::stringstream msg;
msg << "Remote command timed out while waiting to get a connection from the "
<< "pool, took " << getConnectionDuration << ", timeout was set to "
<< timeout;
auto rs = ResponseStatus(ErrorCodes::NetworkInterfaceExceededTimeLimit,
msg.str(),
getConnectionDuration);
return _completeOperation(op, rs);
}
// The above conditional guarantees that the adjusted timeout will never underflow.
MONGO_ASIO_INVARIANT(timeout > getConnectionDuration, "timeout underflowed", op);
const auto adjustedTimeout = timeout - getConnectionDuration;
const auto requestId = op->_request.id;
try {
op->_timeoutAlarm =
op->_owner->_timerFactory->make(&op->_strand, adjustedTimeout);
} catch (std::system_error& e) {
severe() << "Failed to construct timer for AsyncOp: " << e.what();
fassertFailed(40334);
}
std::shared_ptr<AsyncOp::AccessControl> access;
std::size_t generation;
{
stdx::lock_guard<stdx::mutex> lk(op->_access->mutex);
access = op->_access;
generation = access->id;
}
op->_timeoutAlarm->asyncWait(
[this, op, access, generation, requestId, adjustedTimeout](std::error_code ec) {
// We must pass a check for safe access before using op inside the
// callback or we may attempt access on an invalid pointer.
stdx::lock_guard<stdx::mutex> lk(access->mutex);
if (generation != access->id) {
// The operation has been cleaned up, do not access.
return;
}
if (!ec) {
LOG(2) << "Request " << requestId << " timed out"
<< ", adjusted timeout after getting connection from pool was "
<< adjustedTimeout << ", op was " << redact(op->toString());
op->timeOut_inlock();
} else {
LOG(2) << "Failed to time request " << requestId
<< "out: " << ec.message() << ", op was "
<< redact(op->toString());
}
});
}
_beginCommunication(op);
});
};
_connectionPool.get(request.target, request.timeout, nextStep);
return Status::OK();
}
Status AuthzManagerExternalStateMongod::dropCollection(
const NamespaceString& collectionName,
const BSONObj& writeConcern) {
fassertFailed(17096);
}
// Print error message from C runtime, then fassert
int crtDebugCallback(int, char* originalMessage, int*) {
StringData message(originalMessage);
log() << "*** C runtime error: " << message.substr(0, message.find('\n')) << ", terminating";
fassertFailed(17006);
}
void* MemoryMappedFile::map(const char *filenameIn, unsigned long long &length, int options) {
verify( fd == 0 && len == 0 ); // can't open more than once
setFilename(filenameIn);
FileAllocator::get()->allocateAsap( filenameIn, length );
/* big hack here: Babble uses db names with colons. doesn't seem to work on windows. temporary perhaps. */
char filename[256];
strncpy(filename, filenameIn, 255);
filename[255] = 0;
{
size_t len = strlen( filename );
for ( size_t i=len-1; i>=0; i-- ) {
if ( filename[i] == '/' ||
filename[i] == '\\' )
break;
if ( filename[i] == ':' )
filename[i] = '_';
}
}
updateLength( filename, length );
{
DWORD createOptions = FILE_ATTRIBUTE_NORMAL;
if ( options & SEQUENTIAL )
createOptions |= FILE_FLAG_SEQUENTIAL_SCAN;
DWORD rw = GENERIC_READ | GENERIC_WRITE;
fd = CreateFileW(
toWideString(filename).c_str(),
rw, // desired access
FILE_SHARE_WRITE | FILE_SHARE_READ, // share mode
NULL, // security
OPEN_ALWAYS, // create disposition
createOptions , // flags
NULL); // hTempl
if ( fd == INVALID_HANDLE_VALUE ) {
DWORD dosError = GetLastError();
log() << "CreateFileW for " << filename
<< " failed with " << errnoWithDescription( dosError )
<< " (file size is " << length << ")"
<< " in MemoryMappedFile::map"
<< endl;
return 0;
}
}
mapped += length;
{
DWORD flProtect = PAGE_READWRITE; //(options & READONLY)?PAGE_READONLY:PAGE_READWRITE;
maphandle = CreateFileMappingW(fd, NULL, flProtect,
length >> 32 /*maxsizehigh*/,
(unsigned) length /*maxsizelow*/,
NULL/*lpName*/);
if ( maphandle == NULL ) {
DWORD dosError = GetLastError();
log() << "CreateFileMappingW for " << filename
<< " failed with " << errnoWithDescription( dosError )
<< " (file size is " << length << ")"
<< " in MemoryMappedFile::map"
<< endl;
close();
fassertFailed( 16225 );
}
}
void *view = 0;
{
scoped_lock lk(mapViewMutex);
DWORD access = ( options & READONLY ) ? FILE_MAP_READ : FILE_MAP_ALL_ACCESS;
LPVOID thisAddress = getNextMemoryMappedFileLocation( length );
view = MapViewOfFileEx(
maphandle, // file mapping handle
access, // access
0, 0, // file offset, high and low
0, // bytes to map, 0 == all
thisAddress ); // address to place file
if ( view == 0 ) {
DWORD dosError = GetLastError();
log() << "MapViewOfFileEx for " << filename
<< " failed with " << errnoWithDescription( dosError )
<< " (file size is " << length << ")"
<< " in MemoryMappedFile::map"
<< endl;
close();
fassertFailed( 16166 );
}
}
views.push_back(view);
memconcept::is(view, memconcept::concept::memorymappedfile, this->filename(), (unsigned) length);
len = length;
return view;
}
Status AuthzManagerExternalStateMongos::renameCollection(
const NamespaceString& oldName,
const NamespaceString& newName) {
fassertFailed(17107);
}
DiskLoc SimpleRecordStoreV1::_allocFromExistingExtents( OperationContext* txn,
int lenToAlloc ) {
// align size up to a multiple of 4
lenToAlloc = (lenToAlloc + (4-1)) & ~(4-1);
freelistAllocs.increment();
DiskLoc loc;
{
DiskLoc *prev = 0;
DiskLoc *bestprev = 0;
DiskLoc bestmatch;
int bestmatchlen = INT_MAX; // sentinel meaning we haven't found a record big enough
int b = bucket(lenToAlloc);
DiskLoc cur = _details->deletedListEntry(b);
int extra = 5; // look for a better fit, a little.
int chain = 0;
while ( 1 ) {
{ // defensive check
int fileNumber = cur.a();
int fileOffset = cur.getOfs();
if (fileNumber < -1 || fileNumber >= 100000 || fileOffset < 0) {
StringBuilder sb;
sb << "Deleted record list corrupted in collection " << _ns
<< ", bucket " << b
<< ", link number " << chain
<< ", invalid link is " << cur.toString()
<< ", throwing Fatal Assertion";
log() << sb.str() << endl;
fassertFailed(16469);
}
}
if ( cur.isNull() ) {
// move to next bucket. if we were doing "extra", just break
if ( bestmatchlen < INT_MAX )
break;
if ( chain > 0 ) {
// if we looked at things in the right bucket, but they were not suitable
freelistBucketExhausted.increment();
}
b++;
if ( b > MaxBucket ) {
// out of space. alloc a new extent.
freelistIterations.increment( 1 + chain );
return DiskLoc();
}
cur = _details->deletedListEntry(b);
prev = 0;
continue;
}
DeletedRecord *r = drec(cur);
if ( r->lengthWithHeaders() >= lenToAlloc &&
r->lengthWithHeaders() < bestmatchlen ) {
bestmatchlen = r->lengthWithHeaders();
bestmatch = cur;
bestprev = prev;
if (r->lengthWithHeaders() == lenToAlloc)
// exact match, stop searching
break;
}
if ( bestmatchlen < INT_MAX && --extra <= 0 )
break;
if ( ++chain > 30 && b <= MaxBucket ) {
// too slow, force move to next bucket to grab a big chunk
//b++;
freelistIterations.increment( chain );
chain = 0;
cur.Null();
}
else {
cur = r->nextDeleted();
prev = &r->nextDeleted();
}
}
// unlink ourself from the deleted list
DeletedRecord *bmr = drec(bestmatch);
if ( bestprev ) {
*txn->recoveryUnit()->writing(bestprev) = bmr->nextDeleted();
}
else {
// should be the front of a free-list
int myBucket = bucket(bmr->lengthWithHeaders());
invariant( _details->deletedListEntry(myBucket) == bestmatch );
_details->setDeletedListEntry(txn, myBucket, bmr->nextDeleted());
}
*txn->recoveryUnit()->writing(&bmr->nextDeleted()) = DiskLoc().setInvalid(); // defensive.
invariant(bmr->extentOfs() < bestmatch.getOfs());
freelistIterations.increment( 1 + chain );
loc = bestmatch;
}
if ( loc.isNull() )
return loc;
// determine if we should chop up
DeletedRecord *r = drec(loc);
/* note we want to grab from the front so our next pointers on disk tend
to go in a forward direction which is important for performance. */
int regionlen = r->lengthWithHeaders();
invariant( r->extentOfs() < loc.getOfs() );
int left = regionlen - lenToAlloc;
if ( left < 24 || left < (lenToAlloc / 8) ) {
// you get the whole thing.
return loc;
}
// don't quantize:
// - $ collections (indexes) as we already have those aligned the way we want SERVER-8425
if ( _normalCollection ) {
// we quantize here so that it only impacts newly sized records
// this prevents oddities with older records and space re-use SERVER-8435
lenToAlloc = std::min( r->lengthWithHeaders(),
quantizeAllocationSpace( lenToAlloc ) );
left = regionlen - lenToAlloc;
if ( left < 24 ) {
// you get the whole thing.
return loc;
}
}
/* split off some for further use. */
txn->recoveryUnit()->writingInt(r->lengthWithHeaders()) = lenToAlloc;
DiskLoc newDelLoc = loc;
newDelLoc.inc(lenToAlloc);
DeletedRecord* newDel = drec(newDelLoc);
DeletedRecord* newDelW = txn->recoveryUnit()->writing(newDel);
newDelW->extentOfs() = r->extentOfs();
newDelW->lengthWithHeaders() = left;
newDelW->nextDeleted().Null();
addDeletedRec( txn, newDelLoc );
return loc;
}
Status AuthzManagerExternalStateMongos::copyCollection(
const NamespaceString& fromName,
const NamespaceString& toName) {
fassertFailed(17108);
}
void prefetchIndexPages(NamespaceDetails *nsd, const BSONObj& obj) {
DiskLoc unusedDl; // unused
IndexInterface::IndexInserter inserter;
BSONObjSet unusedKeys;
ReplSetImpl::IndexPrefetchConfig prefetchConfig = theReplSet->getIndexPrefetchConfig();
// do we want prefetchConfig to be (1) as-is, (2) for update ops only, or (3) configured per op type?
// One might want PREFETCH_NONE for updates, but it's more rare that it is a bad idea for inserts.
// #3 (per op), a big issue would be "too many knobs".
switch (prefetchConfig) {
case ReplSetImpl::PREFETCH_NONE:
return;
case ReplSetImpl::PREFETCH_ID_ONLY:
{
TimerHolder timer( &prefetchIndexStats);
// on the update op case, the call to prefetchRecordPages will touch the _id index.
// thus perhaps this option isn't very useful?
int indexNo = nsd->findIdIndex();
if (indexNo == -1) return;
try {
fetchIndexInserters(/*out*/unusedKeys,
inserter,
nsd,
indexNo,
obj,
unusedDl,
/*allowDups*/true);
}
catch (const DBException& e) {
LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl;
}
break;
}
case ReplSetImpl::PREFETCH_ALL:
{
// indexCount includes all indexes, including ones
// in the process of being built
int indexCount = nsd->getTotalIndexCount();
for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) {
TimerHolder timer( &prefetchIndexStats);
// This will page in all index pages for the given object.
try {
fetchIndexInserters(/*out*/unusedKeys,
inserter,
nsd,
indexNo,
obj,
unusedDl,
/*allowDups*/true);
}
catch (const DBException& e) {
LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl;
}
unusedKeys.clear();
}
break;
}
default:
fassertFailed(16427);
}
}
void AuthzManagerExternalStateMongos::unlockUpgradeProcess() {
fassertFailed(17110);
}
Status AuthzManagerExternalStateMongod::copyCollection(
const NamespaceString& fromName,
const NamespaceString& toName,
const BSONObj& writeConcern) {
fassertFailed(17098);
}