Exemple #1
0
void ThriftServer::setup() {
  DCHECK_NOTNULL(getProcessorFactory().get());
  auto nWorkers = getNumIOWorkerThreads();
  DCHECK_GT(nWorkers, 0);

  uint32_t threadsStarted = 0;

  // Initialize event base for this thread, ensure event_init() is called
  serveEventBase_ = eventBaseManager_->getEventBase();
  if (idleServerTimeout_.count() > 0) {
    idleServer_.emplace(
        *this, serveEventBase_.load()->timer(), idleServerTimeout_);
  }
  // Print some libevent stats
  VLOG(1) << "libevent " << folly::EventBase::getLibeventVersion() << " method "
          << folly::EventBase::getLibeventMethod();

  try {
#ifndef _WIN32
    // OpenSSL might try to write to a closed socket if the peer disconnects
    // abruptly, raising a SIGPIPE signal. By default this will terminate the
    // process, which we don't want. Hence we need to handle SIGPIPE specially.
    //
    // We don't use SIG_IGN here as child processes will inherit that handler.
    // Instead, we swallow the signal to enable SIGPIPE in children to behave
    // normally.
    // Furthermore, setting flags to 0 and using sigaction prevents SA_RESTART
    // from restarting syscalls after the handler completed. This is important
    // for code using SIGPIPE to interrupt syscalls in other threads.
    struct sigaction sa = {};
    sa.sa_handler = [](int) {};
    sa.sa_flags = 0;
    sigemptyset(&sa.sa_mask);
    sigaction(SIGPIPE, &sa, nullptr);
#endif

    if (!getObserver() && server::observerFactory_) {
      setObserver(server::observerFactory_->getObserver());
    }

    // We always need a threadmanager for cpp2.
    setupThreadManager();
    threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
      EventTask* task = dynamic_cast<EventTask*>(r.get());
      if (task) {
        task->expired();
      }
    });
    threadManager_->setCodelCallback([&](std::shared_ptr<Runnable>) {
      auto observer = getObserver();
      if (observer) {
        if (getEnableCodel()) {
          observer->queueTimeout();
        } else {
          observer->shadowQueueTimeout();
        }
      }
    });
    if (thriftProcessor_) {
      thriftProcessor_->setThreadManager(threadManager_.get());
      thriftProcessor_->setCpp2Processor(getCpp2Processor());
    }

    if (!serverChannel_) {
      ServerBootstrap::socketConfig.acceptBacklog = getListenBacklog();
      ServerBootstrap::socketConfig.maxNumPendingConnectionsPerWorker =
          getMaxNumPendingConnectionsPerWorker();
      if (reusePort_) {
        ServerBootstrap::setReusePort(true);
      }
      if (enableTFO_) {
        ServerBootstrap::socketConfig.enableTCPFastOpen = *enableTFO_;
        ServerBootstrap::socketConfig.fastOpenQueueSize = fastOpenQueueSize_;
      }

      // Resize the IO pool
      ioThreadPool_->setNumThreads(nWorkers);
      if (!acceptPool_) {
        acceptPool_ = std::make_shared<folly::IOThreadPoolExecutor>(
            nAcceptors_,
            std::make_shared<folly::NamedThreadFactory>("Acceptor Thread"));
      }

      // Resize the SSL handshake pool
      size_t nSSLHandshakeWorkers = getNumSSLHandshakeWorkerThreads();
      VLOG(1) << "Using " << nSSLHandshakeWorkers << " SSL handshake threads";
      sslHandshakePool_->setNumThreads(nSSLHandshakeWorkers);

      ServerBootstrap::childHandler(
          acceptorFactory_ ? acceptorFactory_
                           : std::make_shared<ThriftAcceptorFactory>(this));

      {
        std::lock_guard<std::mutex> lock(ioGroupMutex_);
        ServerBootstrap::group(acceptPool_, ioThreadPool_);
      }
      if (socket_) {
        ServerBootstrap::bind(std::move(socket_));
      } else if (port_ != -1) {
        ServerBootstrap::bind(port_);
      } else {
        ServerBootstrap::bind(address_);
      }
      // Update address_ with the address that we are actually bound to.
      // (This is needed if we were supplied a pre-bound socket, or if
      // address_'s port was set to 0, so an ephemeral port was chosen by
      // the kernel.)
      ServerBootstrap::getSockets()[0]->getAddress(&address_);

      for (auto& socket : getSockets()) {
        socket->setShutdownSocketSet(wShutdownSocketSet_);
        socket->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
        try {
          socket->setTosReflect(tosReflect_);
        } catch (std::exception const& ex) {
          LOG(ERROR) << "Got exception setting up TOS reflect: "
                     << folly::exceptionStr(ex);
        }
      }

      // Notify handler of the preServe event
      if (eventHandler_ != nullptr) {
        eventHandler_->preServe(&address_);
      }

    } else {
      startDuplex();
    }

    // Do not allow setters to be called past this point until the IO worker
    // threads have been joined in stopWorkers().
    configMutable_ = false;
  } catch (std::exception& ex) {
    // This block allows us to investigate the exception using gdb
    LOG(ERROR) << "Got an exception while setting up the server: " << ex.what();
    handleSetupFailure();
    throw;
  } catch (...) {
    handleSetupFailure();
    throw;
  }
}
Exemple #2
0
void ThriftServer::setup() {
  DCHECK_NOTNULL(cpp2Pfac_.get());
  DCHECK_GT(nWorkers_, 0);

  uint32_t threadsStarted = 0;
  bool eventBaseAttached = false;

  // Make sure EBM exists if we haven't set one explicitly
  getEventBaseManager();

  // Initialize event base for this thread, ensure event_init() is called
  serveEventBase_ = eventBaseManager_->getEventBase();
  // Print some libevent stats
  LOG(INFO) << "libevent " <<
    TEventBase::getLibeventVersion() << " method " <<
    TEventBase::getLibeventMethod();

  try {
    // We check for write success so we don't need or want SIGPIPEs.
    signal(SIGPIPE, SIG_IGN);

    if (!observer_ && apache::thrift::observerFactory_) {
      observer_ = apache::thrift::observerFactory_->getObserver();
    }

    // bind to the socket
    if (socket_ == nullptr) {
      socket_.reset(new TAsyncServerSocket());
      socket_->setShutdownSocketSet(shutdownSocketSet_.get());
      if (port_ != -1) {
        socket_->bind(port_);
      } else {
        DCHECK(address_.isInitialized());
        socket_->bind(address_);
      }
    }

    socket_->listen(listenBacklog_);
    socket_->setMaxNumMessagesInQueue(maxNumMsgsInQueue_);
    socket_->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);

    // We always need a threadmanager for cpp2.
    if (!threadFactory_) {
      setThreadFactory(std::shared_ptr<ThreadFactory>(
        new PosixThreadFactory));
    }

    if (!threadManager_) {
      std::shared_ptr<apache::thrift::concurrency::ThreadManager>
        threadManager(PriorityThreadManager::newPriorityThreadManager(
                        nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_,
                        true /*stats*/));
      threadManager->enableCodel(getEnableCodel());
      threadManager->start();
      setThreadManager(threadManager);
    }
    threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
        EventTask* task = dynamic_cast<EventTask*>(r.get());
        if (task) {
          task->expired();
        }
    });
    threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
        auto observer = getObserver();
        if (observer) {
          observer->queueTimeout();
        }
    });

    auto b = std::make_shared<boost::barrier>(nWorkers_ + 1);

    // Create the worker threads.
    workers_.reserve(nWorkers_);
    for (uint32_t n = 0; n < nWorkers_; ++n) {
      addWorker();
      workers_[n].worker->getEventBase()->runInLoop([b](){
        b->wait();
      });
    }

    // Update address_ with the address that we are actually bound to.
    // (This is needed if we were supplied a pre-bound socket, or if address_'s
    // port was set to 0, so an ephemeral port was chosen by the kernel.)
    socket_->getAddress(&address_);

    // Notify handler of the preServe event
    if (eventHandler_ != nullptr) {
      eventHandler_->preServe(&address_);
    }

    for (auto& worker: workers_) {
      worker.thread->start();
      ++threadsStarted;
      worker.thread->setName(folly::to<std::string>("Cpp2Worker",
                                                    threadsStarted));
    }

    // Wait for all workers to start
    b->wait();

    socket_->attachEventBase(eventBaseManager_->getEventBase());
    eventBaseAttached = true;
    socket_->startAccepting();
  } catch (...) {
    // XXX: Cpp2Worker::acceptStopped() calls
    //      eventBase_.terminateLoopSoon().  Normally this stops the
    //      worker from processing more work and stops the event loop.
    //      However, if startConsuming() and eventBase_.loop() haven't
    //      run yet this won't do the right thing. The worker thread
    //      will still continue to call startConsuming() later, and
    //      will then start the event loop.
    for (uint32_t i = 0; i < threadsStarted; ++i) {
      workers_[i].worker->acceptStopped();
      workers_[i].thread->join();
    }
    workers_.clear();

    if (socket_) {
      if (eventBaseAttached) {
        socket_->detachEventBase();
      }

      socket_.reset();
    }

    throw;
  }
}
Exemple #3
0
void ThriftServer::setup() {
  DCHECK_NOTNULL(cpp2Pfac_.get());
  DCHECK_GT(nWorkers_, 0);

  uint32_t threadsStarted = 0;

  // Initialize event base for this thread, ensure event_init() is called
  serveEventBase_ = eventBaseManager_->getEventBase();
  // Print some libevent stats
  VLOG(1) << "libevent " <<
    folly::EventBase::getLibeventVersion() << " method " <<
    folly::EventBase::getLibeventMethod();

  try {
    // We check for write success so we don't need or want SIGPIPEs.
    signal(SIGPIPE, SIG_IGN);

    if (!observer_ && apache::thrift::observerFactory_) {
      observer_ = apache::thrift::observerFactory_->getObserver();
    }

    // We always need a threadmanager for cpp2.
    if (!threadFactory_) {
      setThreadFactory(
        std::make_shared<apache::thrift::concurrency::PosixThreadFactory>(
          apache::thrift::concurrency::PosixThreadFactory::kDefaultPolicy,
          apache::thrift::concurrency::PosixThreadFactory::kDefaultPriority,
          threadStackSizeMB_
        )
      );
    }

    if (saslPolicy_ == "required" || saslPolicy_ == "permitted") {
      if (!saslThreadManager_) {
        auto numThreads = nSaslPoolThreads_ > 0
                              ? nSaslPoolThreads_
                              : (nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_);
        saslThreadManager_ = ThreadManager::newSimpleThreadManager(
            numThreads,
            0, /* pendingTaskCountMax -- no limit */
            false, /* enableTaskStats */
            0 /* maxQueueLen -- large default */);
        saslThreadManager_->setNamePrefix("thrift-sasl");
        saslThreadManager_->threadFactory(threadFactory_);
        saslThreadManager_->start();
      }
      auto saslThreadManager = saslThreadManager_;

      if (getSaslServerFactory()) {
        // If the factory is already set, don't override it with the default
      } else if (FLAGS_pin_service_identity &&
                 !FLAGS_service_identity.empty()) {
        // If pin_service_identity flag is set and service_identity is specified
        // force the server use the corresponding principal from keytab.
        char hostname[256];
        if (gethostname(hostname, 255)) {
          LOG(FATAL) << "Failed getting hostname";
        }
        setSaslServerFactory([=] (folly::EventBase* evb) {
          auto saslServer = std::unique_ptr<SaslServer>(
            new GssSaslServer(evb, saslThreadManager));
          saslServer->setServiceIdentity(
            FLAGS_service_identity + "/" + hostname);
          return saslServer;
        });
      } else {
        // Allow the server to accept anything in the keytab.
        setSaslServerFactory([=] (folly::EventBase* evb) {
          return std::unique_ptr<SaslServer>(
            new GssSaslServer(evb, saslThreadManager));
        });
      }
    }

    if (!threadManager_) {
      int numThreads = nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_;
      std::shared_ptr<apache::thrift::concurrency::ThreadManager>
        threadManager(PriorityThreadManager::newPriorityThreadManager(
                        numThreads,
                        true /*stats*/,
                        getMaxRequests() + numThreads /*maxQueueLen*/));
      threadManager->enableCodel(getEnableCodel());
      if (!poolThreadName_.empty()) {
        threadManager->setNamePrefix(poolThreadName_);
      }
      threadManager->start();
      setThreadManager(threadManager);
    }
    threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
        EventTask* task = dynamic_cast<EventTask*>(r.get());
        if (task) {
          task->expired();
        }
    });
    threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
        auto observer = getObserver();
        if (observer) {
          observer->queueTimeout();
        }
    });

    if (!serverChannel_) {

      ServerBootstrap::socketConfig.acceptBacklog = listenBacklog_;

      // Resize the IO pool
      ioThreadPool_->setNumThreads(nWorkers_);

      ServerBootstrap::childHandler(
          acceptorFactory_ ? acceptorFactory_
                           : std::make_shared<ThriftAcceptorFactory>(this));

      {
        std::lock_guard<std::mutex> lock(ioGroupMutex_);
        ServerBootstrap::group(acceptPool_, ioThreadPool_);
      }
      if (socket_) {
        ServerBootstrap::bind(std::move(socket_));
      } else if (port_ != -1) {
        ServerBootstrap::bind(port_);
      } else {
        ServerBootstrap::bind(address_);
      }
      // Update address_ with the address that we are actually bound to.
      // (This is needed if we were supplied a pre-bound socket, or if
      // address_'s port was set to 0, so an ephemeral port was chosen by
      // the kernel.)
      ServerBootstrap::getSockets()[0]->getAddress(&address_);

      for (auto& socket : getSockets()) {
        socket->setShutdownSocketSet(shutdownSocketSet_.get());
        socket->setMaxNumMessagesInQueue(maxNumPendingConnectionsPerWorker_);
        socket->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
      }

      // Notify handler of the preServe event
      if (eventHandler_ != nullptr) {
        eventHandler_->preServe(&address_);
      }

    } else {
      CHECK(configMutable());
      duplexWorker_ = folly::make_unique<Cpp2Worker>(this, serverChannel_);
      // we don't control the EventBase for the duplexWorker, so when we shut
      // it down, we need to ensure there's no delay
      duplexWorker_->setGracefulShutdownTimeout(std::chrono::milliseconds(0));
    }

    // Do not allow setters to be called past this point until the IO worker
    // threads have been joined in stopWorkers().
    configMutable_ = false;
  } catch (std::exception& ex) {
    // This block allows us to investigate the exception using gdb
    LOG(ERROR) << "Got an exception while setting up the server: "
      << ex.what();
    handleSetupFailure();
    throw;
  } catch (...) {
    handleSetupFailure();
    throw;
  }
}
Exemple #4
0
void ThriftServer::setup() {
  DCHECK_NOTNULL(cpp2Pfac_.get());
  DCHECK_GT(nWorkers_, 0);

  uint32_t threadsStarted = 0;
  bool eventBaseAttached = false;

  // Make sure EBM exists if we haven't set one explicitly
  getEventBaseManager();

  // Initialize event base for this thread, ensure event_init() is called
  serveEventBase_ = eventBaseManager_->getEventBase();
  // Print some libevent stats
  LOG(INFO) << "libevent " <<
    TEventBase::getLibeventVersion() << " method " <<
    TEventBase::getLibeventMethod();

  try {
    // We check for write success so we don't need or want SIGPIPEs.
    signal(SIGPIPE, SIG_IGN);

    if (!observer_ && apache::thrift::observerFactory_) {
      observer_ = apache::thrift::observerFactory_->getObserver();
    }

    // bind to the socket
    if (!serverChannel_) {
      if (socket_ == nullptr) {
        socket_.reset(new TAsyncServerSocket());
        socket_->setShutdownSocketSet(shutdownSocketSet_.get());
        if (port_ != -1) {
          socket_->bind(port_);
        } else {
          DCHECK(address_.isInitialized());
          socket_->bind(address_);
        }
      }

      socket_->listen(listenBacklog_);
      socket_->setMaxNumMessagesInQueue(maxNumMsgsInQueue_);
      socket_->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
    }

    // We always need a threadmanager for cpp2.
    if (!threadFactory_) {
      setThreadFactory(
        std::make_shared<apache::thrift::concurrency::NumaThreadFactory>());
    }

    if (FLAGS_sasl_policy == "required" || FLAGS_sasl_policy == "permitted") {
      if (!saslThreadManager_) {
        saslThreadManager_ = ThreadManager::newSimpleThreadManager(
          nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_, /* count */
          0, /* pendingTaskCountMax -- no limit */
          false, /* enableTaskStats */
          0 /* maxQueueLen -- large default */);
        saslThreadManager_->setNamePrefix("thrift-sasl");
        saslThreadManager_->threadFactory(threadFactory_);
        saslThreadManager_->start();
      }
      auto saslThreadManager = saslThreadManager_;

      if (getSaslServerFactory()) {
        // If the factory is already set, don't override it with the default
      } else if (FLAGS_kerberos_service_name.empty()) {
        // If the service name is not specified, not need to pin the principal.
        // Allow the server to accept anything in the keytab.
        setSaslServerFactory([=] (TEventBase* evb) {
          return std::unique_ptr<SaslServer>(
            new GssSaslServer(evb, saslThreadManager));
        });
      } else {
        char hostname[256];
        if (gethostname(hostname, 255)) {
          LOG(FATAL) << "Failed getting hostname";
        }
        setSaslServerFactory([=] (TEventBase* evb) {
          auto saslServer = std::unique_ptr<SaslServer>(
            new GssSaslServer(evb, saslThreadManager));
          saslServer->setServiceIdentity(
            FLAGS_kerberos_service_name + "/" + hostname);
          return std::move(saslServer);
        });
      }
    }

    if (!threadManager_) {
      std::shared_ptr<apache::thrift::concurrency::ThreadManager>
        threadManager(new apache::thrift::concurrency::NumaThreadManager(
                        nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_,
                        true /*stats*/,
                        getMaxRequests() /*maxQueueLen*/));
      threadManager->enableCodel(getEnableCodel());
      if (!poolThreadName_.empty()) {
        threadManager->setNamePrefix(poolThreadName_);
      }
      threadManager->start();
      setThreadManager(threadManager);
    }
    threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
        EventTask* task = dynamic_cast<EventTask*>(r.get());
        if (task) {
          task->expired();
        }
    });
    threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
        auto observer = getObserver();
        if (observer) {
          observer->queueTimeout();
        }
    });

    if (!serverChannel_) {
      // regular server
      auto b = std::make_shared<boost::barrier>(nWorkers_ + 1);

      // Create the worker threads.
      workers_.reserve(nWorkers_);
      for (uint32_t n = 0; n < nWorkers_; ++n) {
        addWorker();
        workers_[n].worker->getEventBase()->runInLoop([b](){
          b->wait();
        });
      }

      // Update address_ with the address that we are actually bound to.
      // (This is needed if we were supplied a pre-bound socket, or if
      // address_'s port was set to 0, so an ephemeral port was chosen by
      // the kernel.)
      if (socket_) {
        socket_->getAddress(&address_);
      }

      // Notify handler of the preServe event
      if (eventHandler_ != nullptr) {
        eventHandler_->preServe(&address_);
      }

      for (auto& worker: workers_) {
        worker.thread->start();
        ++threadsStarted;
        worker.thread->setName(folly::to<std::string>(cpp2WorkerThreadName_,
                                                      threadsStarted));
      }

      // Wait for all workers to start
      b->wait();

      if (socket_) {
        socket_->attachEventBase(eventBaseManager_->getEventBase());
      }
      eventBaseAttached = true;
      if (socket_) {
        socket_->startAccepting();
      }
    } else {
      // duplex server
      // Create the Cpp2Worker
      DCHECK(workers_.empty());
      WorkerInfo info;
      uint32_t workerID = 0;
      info.worker.reset(new Cpp2Worker(this, workerID, serverChannel_));
      // no thread, use current one (shared with client)
      info.thread = nullptr;

      workers_.push_back(info);
    }
  } catch (...) {
    // XXX: Cpp2Worker::acceptStopped() calls
    //      eventBase_.terminateLoopSoon().  Normally this stops the
    //      worker from processing more work and stops the event loop.
    //      However, if startConsuming() and eventBase_.loop() haven't
    //      run yet this won't do the right thing. The worker thread
    //      will still continue to call startConsuming() later, and
    //      will then start the event loop.
    if (!serverChannel_) {
      for (uint32_t i = 0; i < threadsStarted; ++i) {
        workers_[i].worker->acceptStopped();
        workers_[i].thread->join();
      }
    }
    workers_.clear();

    if (socket_) {
      if (eventBaseAttached) {
        socket_->detachEventBase();
      }

      socket_.reset();
    }

    // avoid crash on stop()
    serveEventBase_ = nullptr;

    throw;
  }
}