Exemple #1
0
// NOTE: This function may be called at any time after
// registerShutdownTask is called below. It must not depend on the
// prior execution of mongo initializers or the existence of threads.
static void cleanupTask() {
    {
        auto serviceContext = getGlobalServiceContext();
        Client::initThreadIfNotAlready();
        Client& client = cc();
        ServiceContext::UniqueOperationContext uniqueTxn;
        OperationContext* txn = client.getOperationContext();
        if (!txn) {
            uniqueTxn = client.makeOperationContext();
            txn = uniqueTxn.get();
        }

        if (serviceContext)
            serviceContext->setKillAllOperations();

        if (auto cursorManager = Grid::get(txn)->getCursorManager()) {
            cursorManager->shutdown();
        }
        if (auto pool = Grid::get(txn)->getExecutorPool()) {
            pool->shutdownAndJoin();
        }
        if (auto catalog = Grid::get(txn)->catalogClient(txn)) {
            catalog->shutDown(txn);
        }
    }

    audit::logShutdown(Client::getCurrent());
}
Exemple #2
0
void TaskRunner::_runTasks() {
    // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() to be equal to the
    // client used to create the operation context.
    Client* client = &cc();
    if (AuthorizationManager::get(client->getServiceContext())->isAuthEnabled()) {
        AuthorizationSession::get(client)->grantInternalAuthorization(client);
    }
    ServiceContext::UniqueOperationContext opCtx;

    while (Task task = _waitForNextTask()) {
        if (!opCtx) {
            opCtx = client->makeOperationContext();
        }

        NextAction nextAction = runSingleTask(task, opCtx.get(), Status::OK());

        if (nextAction != NextAction::kKeepOperationContext) {
            opCtx.reset();
        }

        if (nextAction == NextAction::kCancel) {
            break;
        }
        // Release thread back to pool after disposing if no scheduled tasks in queue.
        if (nextAction == NextAction::kDisposeOperationContext ||
            nextAction == NextAction::kInvalid) {
            stdx::lock_guard<stdx::mutex> lk(_mutex);
            if (_tasks.empty()) {
                _finishRunTasks_inlock();
                return;
            }
        }
    }
    opCtx.reset();

    std::list<Task> tasks;
    UniqueLock lk{_mutex};

    auto cancelTasks = [&]() {
        tasks.swap(_tasks);
        lk.unlock();
        // Cancel remaining tasks with a CallbackCanceled status.
        for (auto&& task : tasks) {
            runSingleTask(std::move(task),
                          nullptr,
                          Status(ErrorCodes::CallbackCanceled,
                                 "this task has been canceled by a previously invoked task"));
        }
        tasks.clear();

    };
    cancelTasks();

    lk.lock();
    _finishRunTasks_inlock();
    cancelTasks();
}
Exemple #3
0
void TaskRunner::_runTasks() {
    Client* client = nullptr;
    ServiceContext::UniqueOperationContext txn;

    while (Task task = _waitForNextTask()) {
        if (!txn) {
            if (!client) {
                // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc()
                // to be equal to the client used to create the operation context.
                Client::initThreadIfNotAlready();
                client = &cc();
                if (getGlobalAuthorizationManager()->isAuthEnabled()) {
                    AuthorizationSession::get(client)->grantInternalAuthorization();
                }
            }
            txn = client->makeOperationContext();
        }

        NextAction nextAction = runSingleTask(task, txn.get(), Status::OK());

        if (nextAction != NextAction::kKeepOperationContext) {
            txn.reset();
        }

        if (nextAction == NextAction::kCancel) {
            break;
        }
        // Release thread back to pool after disposing if no scheduled tasks in queue.
        if (nextAction == NextAction::kDisposeOperationContext ||
            nextAction == NextAction::kInvalid) {
            stdx::lock_guard<stdx::mutex> lk(_mutex);
            if (_tasks.empty()) {
                _finishRunTasks_inlock();
                return;
            }
        }
    }
    txn.reset();

    std::list<Task> tasks;
    {
        stdx::lock_guard<stdx::mutex> lk(_mutex);
        tasks.swap(_tasks);
    }

    // Cancel remaining tasks with a CallbackCanceled status.
    for (auto task : tasks) {
        runSingleTask(task,
                      nullptr,
                      Status(ErrorCodes::CallbackCanceled,
                             "this task has been canceled by a previously invoked task"));
    }

    stdx::lock_guard<stdx::mutex> lk(_mutex);
    _finishRunTasks_inlock();
}
void DatabasesCloner::_onEachDBCloneFinish(const Status& status, const std::string& name) {
    UniqueLock lk(_mutex);
    if (!status.isOK()) {
        warning() << "database '" << name << "' (" << (_stats.databasesCloned + 1) << " of "
                  << _databaseCloners.size() << ") clone failed due to " << status.toString();
        _fail_inlock(&lk, status);
        return;
    }

    if (StringData(name).equalCaseInsensitive("admin")) {
        LOG(1) << "Finished the 'admin' db, now calling isAdminDbValid.";
        // Do special checks for the admin database because of auth. collections.
        auto adminStatus = Status(ErrorCodes::NotYetInitialized, "");
        {
            // TODO: Move isAdminDbValid() out of the collection/database cloner code paths.
            OperationContext* opCtx = cc().getOperationContext();
            ServiceContext::UniqueOperationContext opCtxPtr;
            if (!opCtx) {
                opCtxPtr = cc().makeOperationContext();
                opCtx = opCtxPtr.get();
            }
            adminStatus = _storage->isAdminDbValid(opCtx);
        }
        if (!adminStatus.isOK()) {
            LOG(1) << "Validation failed on 'admin' db due to " << adminStatus;
            _fail_inlock(&lk, adminStatus);
            return;
        }
    }

    _stats.databasesCloned++;

    if (_stats.databasesCloned == _databaseCloners.size()) {
        _succeed_inlock(&lk);
        return;
    }

    // Start next database cloner.
    auto&& dbCloner = _databaseCloners[_stats.databasesCloned];
    auto startStatus = dbCloner->startup();
    if (!startStatus.isOK()) {
        warning() << "failed to schedule database '" << name << "' ("
                  << (_stats.databasesCloned + 1) << " of " << _databaseCloners.size()
                  << ") due to " << startStatus.toString();
        _fail_inlock(&lk, startStatus);
        return;
    }
}
Exemple #5
0
    void run() {
        const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
        OperationContext& opCtx = *opCtxPtr;
        const NamespaceString nss("unittests.matchertests");
        AutoGetCollectionForReadCommand ctx(&opCtx, nss);

        const CollatorInterface* collator = nullptr;
        const boost::intrusive_ptr<ExpressionContext> expCtx(
            new ExpressionContext(opCtxPtr.get(), collator));
        M m(BSON("$where"
                 << "function(){ return this.a == 1; }"),
            expCtx,
            ExtensionsCallbackReal(&opCtx, &nss),
            MatchExpressionParser::AllowedFeatures::kJavascript);
        ASSERT(m.matches(BSON("a" << 1)));
        ASSERT(!m.matches(BSON("a" << 2)));
    }
Exemple #6
0
// NOTE: This function may be called at any time after
// registerShutdownTask is called below. It must not depend on the
// prior execution of mongo initializers or the existence of threads.
static void cleanupTask() {
    {
        Client& client = cc();
        ServiceContext::UniqueOperationContext uniqueTxn;
        OperationContext* txn = client.getOperationContext();
        if (!txn) {
            uniqueTxn = client.makeOperationContext();
            txn = uniqueTxn.get();
        }

        auto cursorManager = grid.getCursorManager();
        cursorManager->shutdown();
        grid.shardRegistry()->shutdown();
        grid.catalogManager(txn)->shutDown(txn);
    }

    audit::logShutdown(ClientBasic::getCurrent());
}
Exemple #7
0
// NOTE: This function may be called at any time after
// registerShutdownTask is called below. It must not depend on the
// prior execution of mongo initializers or the existence of threads.
static void cleanupTask() {
    {
        Client::initThreadIfNotAlready();
        Client& client = cc();
        ServiceContext::UniqueOperationContext uniqueTxn;
        OperationContext* txn = client.getOperationContext();
        if (!txn) {
            uniqueTxn = client.makeOperationContext();
            txn = uniqueTxn.get();
        }

        auto cursorManager = grid.getCursorManager();
        cursorManager->shutdown();
        grid.getExecutorPool()->shutdownAndJoin();
        grid.catalogClient(txn)->shutDown(txn);
    }

    audit::logShutdown(Client::getCurrent());
}
Exemple #8
0
/**
 * The main durability thread loop. There is a single instance of this function running.
 */
static void durThread(ClockSource* cs, int64_t serverStartMs) {
    Client::initThread("durability");

    log() << "Durability thread started";

    bool samePartition = true;
    try {
        const std::string dbpathDir = boost::filesystem::path(storageGlobalParams.dbpath).string();
        samePartition = onSamePartition(getJournalDir().string(), dbpathDir);
    } catch (...) {
    }

    // Spawn the journal writer thread
    JournalWriter journalWriter(&commitNotify, &applyToDataFilesNotify, NumAsyncJournalWrites);
    journalWriter.start();

    // Used as an estimate of how much / how fast to remap
    uint64_t commitCounter(0);
    uint64_t estimatedPrivateMapSize(0);
    uint64_t remapLastTimestamp(0);

    while (shutdownRequested.loadRelaxed() == 0) {
        unsigned ms = storageGlobalParams.journalCommitIntervalMs.load();
        if (ms == 0) {
            ms = samePartition ? 100 : 30;
        }

        // +1 so it never goes down to zero
        const int64_t oneThird = (ms / 3) + 1;

        // Reset the stats based on the reset interval
        if (stats.curr()->getCurrentDurationMillis() > DurStatsResetIntervalMillis) {
            stats.reset();
        }

        try {
            stdx::unique_lock<stdx::mutex> lock(flushMutex);

            for (unsigned i = 0; i <= 2; i++) {
                if (stdx::cv_status::no_timeout ==
                    flushRequested.wait_for(lock, Milliseconds(oneThird).toSystemDuration())) {
                    // Someone forced a flush
                    break;
                }

                if (commitNotify.nWaiting()) {
                    // One or more getLastError j:true is pending
                    break;
                }

                if (commitJob.bytes() > UncommittedBytesLimit / 2) {
                    // The number of written bytes is growing
                    break;
                }
            }

            // The commit logic itself
            LOG(4) << "groupCommit begin";

            Timer t;

            const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
            OperationContext& opCtx = *opCtxPtr;
            AutoAcquireFlushLockForMMAPV1Commit autoFlushLock(opCtx.lockState());

            // We need to snapshot the commitNumber after the flush lock has been obtained,
            // because at this point we know that we have a stable snapshot of the data.
            const CommitNotifier::When commitNumber(commitNotify.now());

            LOG(4) << "Processing commit number " << commitNumber;

            if (!commitJob.hasWritten()) {
                // We do not need the journal lock anymore. Free it here, for the really
                // unlikely possibility that the writeBuffer command below blocks.
                autoFlushLock.release();

                // getlasterror request could have came after the data was already committed.
                // No need to call committingReset though, because we have not done any
                // writes (hasWritten == false).
                JournalWriter::Buffer* const buffer = journalWriter.newBuffer();
                buffer->setNoop();
                buffer->journalListenerToken = getJournalListener()->getToken();

                journalWriter.writeBuffer(buffer, commitNumber);
            } else {
                // This copies all the in-memory changes into the journal writer's buffer.
                JournalWriter::Buffer* const buffer = journalWriter.newBuffer();
                PREPLOGBUFFER(buffer->getHeader(), buffer->getBuilder(), cs, serverStartMs);

                estimatedPrivateMapSize += commitJob.bytes();
                commitCounter++;

                // Now that the write intents have been copied to the buffer, the commit job is
                // free to be reused. We need to reset the commit job's contents while under
                // the S flush lock, because otherwise someone might have done a write and this
                // would wipe out their changes without ever being committed.
                commitJob.committingReset();

                double systemMemoryPressurePercentage =
                    ProcessInfo::getSystemMemoryPressurePercentage();

                // Now that the in-memory modifications have been collected, we can potentially
                // release the flush lock if remap is not necessary.
                // When we remap due to memory pressure, we look at two criteria
                // 1. If the amount of 4k pages touched exceeds 512 MB,
                //    a reasonable estimate of memory pressure on Linux.
                // 2. Check if the amount of free memory on the machine is running low,
                //    since #1 is underestimates the memory pressure on Windows since
                //    commits in 64MB chunks.
                const bool shouldRemap = (estimatedPrivateMapSize >= UncommittedBytesLimit) ||
                    (systemMemoryPressurePercentage > 0.0) ||
                    (commitCounter % NumCommitsBeforeRemap == 0) ||
                    (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalAlwaysRemap);

                double remapFraction = 0.0;

                if (shouldRemap) {
                    // We want to remap all private views about every 2 seconds. There could be
                    // ~1000 views so we do a little each pass. There will be copy on write
                    // faults after remapping, so doing a little bit at a time will avoid big
                    // load spikes when the pages are touched.
                    //
                    // TODO: Instead of the time-based logic above, consider using ProcessInfo
                    //       and watching for getResidentSize to drop, which is more precise.
                    remapFraction = (curTimeMicros64() - remapLastTimestamp) / 2000000.0;

                    if (mmapv1GlobalOptions.journalOptions & MMAPV1Options::JournalAlwaysRemap) {
                        remapFraction = 1;
                    } else {
                        // We don't want to get close to the UncommittedBytesLimit
                        const double remapMemFraction =
                            estimatedPrivateMapSize / ((double)UncommittedBytesLimit);

                        remapFraction = std::max(remapMemFraction, remapFraction);

                        remapFraction = std::max(systemMemoryPressurePercentage, remapFraction);
                    }
                } else {
                    LOG(4) << "Early release flush lock";

                    // We will not be doing a remap so drop the flush lock. That way we will be
                    // doing the journal I/O outside of lock, so other threads can proceed.
                    invariant(!shouldRemap);
                    autoFlushLock.release();
                }

                buffer->journalListenerToken = getJournalListener()->getToken();
                // Request async I/O to the journal. This may block.
                journalWriter.writeBuffer(buffer, commitNumber);

                // Data has now been written to the shared view. If remap was requested, we
                // would still be holding the S flush lock here, so just upgrade it and
                // perform the remap.
                if (shouldRemap) {
                    // Need to wait for the previously scheduled journal writes to complete
                    // before any remap is attempted.
                    journalWriter.flush();
                    journalWriter.assertIdle();

                    // Upgrading the journal lock to flush stops all activity on the system,
                    // because we will be remapping memory and we don't want readers to be
                    // accessing it. Technically this step could be avoided on systems, which
                    // support atomic remap.
                    autoFlushLock.upgradeFlushLockToExclusive();
                    remapPrivateView(opCtxPtr.get(), remapFraction);

                    autoFlushLock.release();

                    // Reset the private map estimate outside of the lock
                    estimatedPrivateMapSize = 0;
                    remapLastTimestamp = curTimeMicros64();

                    stats.curr()->_commitsInWriteLock++;
                    stats.curr()->_commitsInWriteLockMicros += t.micros();
                }
            }

            stats.curr()->_commits++;
            stats.curr()->_commitsMicros += t.micros();

            LOG(4) << "groupCommit end";
        } catch (DBException& e) {
            severe() << "dbexception in durThread causing immediate shutdown: " << redact(e);
            invariant(false);
        } catch (std::ios_base::failure& e) {
            severe() << "ios_base exception in durThread causing immediate shutdown: "
                     << redact(e.what());
            invariant(false);
        } catch (std::bad_alloc& e) {
            severe() << "bad_alloc exception in durThread causing immediate shutdown: "
                     << redact(e.what());
            invariant(false);
        } catch (std::exception& e) {
            severe() << "exception in durThread causing immediate shutdown: " << redact(e.what());
            invariant(false);
        } catch (...) {
            severe() << "unhandled exception in durThread causing immediate shutdown";
            invariant(false);
        }
    }

    // Stops the journal thread and ensures everything was written
    invariant(!commitJob.hasWritten());

    journalWriter.flush();
    journalWriter.shutdown();

    log() << "Durability thread stopped";
}