void ThriftServer::setupThreadManager() {
if (!threadManager_) {
auto nPoolThreads = getNumCPUWorkerThreads();
int numThreads = nPoolThreads > 0 ? nPoolThreads : getNumIOWorkerThreads();
std::shared_ptr<apache::thrift::concurrency::ThreadManager> threadManager(
PriorityThreadManager::newPriorityThreadManager(
numThreads, true /*stats*/));
threadManager->enableCodel(getEnableCodel());
// If a thread factory has been specified, use it.
if (threadFactory_) {
threadManager->threadFactory(threadFactory_);
}
auto poolThreadName = getCPUWorkerThreadName();
if (!poolThreadName.empty()) {
threadManager->setNamePrefix(poolThreadName);
}
threadManager->start();
setThreadManager(threadManager);
}
}
void ThriftServer::setup() {
DCHECK_NOTNULL(getProcessorFactory().get());
auto nWorkers = getNumIOWorkerThreads();
DCHECK_GT(nWorkers, 0);
uint32_t threadsStarted = 0;
// Initialize event base for this thread, ensure event_init() is called
serveEventBase_ = eventBaseManager_->getEventBase();
if (idleServerTimeout_.count() > 0) {
idleServer_.emplace(
*this, serveEventBase_.load()->timer(), idleServerTimeout_);
}
// Print some libevent stats
VLOG(1) << "libevent " << folly::EventBase::getLibeventVersion() << " method "
<< folly::EventBase::getLibeventMethod();
try {
#ifndef _WIN32
// OpenSSL might try to write to a closed socket if the peer disconnects
// abruptly, raising a SIGPIPE signal. By default this will terminate the
// process, which we don't want. Hence we need to handle SIGPIPE specially.
//
// We don't use SIG_IGN here as child processes will inherit that handler.
// Instead, we swallow the signal to enable SIGPIPE in children to behave
// normally.
// Furthermore, setting flags to 0 and using sigaction prevents SA_RESTART
// from restarting syscalls after the handler completed. This is important
// for code using SIGPIPE to interrupt syscalls in other threads.
struct sigaction sa = {};
sa.sa_handler = [](int) {};
sa.sa_flags = 0;
sigemptyset(&sa.sa_mask);
sigaction(SIGPIPE, &sa, nullptr);
#endif
if (!getObserver() && server::observerFactory_) {
setObserver(server::observerFactory_->getObserver());
}
// We always need a threadmanager for cpp2.
setupThreadManager();
threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
EventTask* task = dynamic_cast<EventTask*>(r.get());
if (task) {
task->expired();
}
});
threadManager_->setCodelCallback([&](std::shared_ptr<Runnable>) {
auto observer = getObserver();
if (observer) {
if (getEnableCodel()) {
observer->queueTimeout();
} else {
observer->shadowQueueTimeout();
}
}
});
if (thriftProcessor_) {
thriftProcessor_->setThreadManager(threadManager_.get());
thriftProcessor_->setCpp2Processor(getCpp2Processor());
}
if (!serverChannel_) {
ServerBootstrap::socketConfig.acceptBacklog = getListenBacklog();
ServerBootstrap::socketConfig.maxNumPendingConnectionsPerWorker =
getMaxNumPendingConnectionsPerWorker();
if (reusePort_) {
ServerBootstrap::setReusePort(true);
}
if (enableTFO_) {
ServerBootstrap::socketConfig.enableTCPFastOpen = *enableTFO_;
ServerBootstrap::socketConfig.fastOpenQueueSize = fastOpenQueueSize_;
}
// Resize the IO pool
ioThreadPool_->setNumThreads(nWorkers);
if (!acceptPool_) {
acceptPool_ = std::make_shared<folly::IOThreadPoolExecutor>(
nAcceptors_,
std::make_shared<folly::NamedThreadFactory>("Acceptor Thread"));
}
// Resize the SSL handshake pool
size_t nSSLHandshakeWorkers = getNumSSLHandshakeWorkerThreads();
VLOG(1) << "Using " << nSSLHandshakeWorkers << " SSL handshake threads";
sslHandshakePool_->setNumThreads(nSSLHandshakeWorkers);
ServerBootstrap::childHandler(
acceptorFactory_ ? acceptorFactory_
: std::make_shared<ThriftAcceptorFactory>(this));
{
std::lock_guard<std::mutex> lock(ioGroupMutex_);
ServerBootstrap::group(acceptPool_, ioThreadPool_);
}
if (socket_) {
ServerBootstrap::bind(std::move(socket_));
} else if (port_ != -1) {
ServerBootstrap::bind(port_);
} else {
ServerBootstrap::bind(address_);
}
// Update address_ with the address that we are actually bound to.
// (This is needed if we were supplied a pre-bound socket, or if
// address_'s port was set to 0, so an ephemeral port was chosen by
// the kernel.)
ServerBootstrap::getSockets()[0]->getAddress(&address_);
for (auto& socket : getSockets()) {
socket->setShutdownSocketSet(wShutdownSocketSet_);
socket->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
try {
socket->setTosReflect(tosReflect_);
} catch (std::exception const& ex) {
LOG(ERROR) << "Got exception setting up TOS reflect: "
<< folly::exceptionStr(ex);
}
}
// Notify handler of the preServe event
if (eventHandler_ != nullptr) {
eventHandler_->preServe(&address_);
}
} else {
startDuplex();
}
// Do not allow setters to be called past this point until the IO worker
// threads have been joined in stopWorkers().
configMutable_ = false;
} catch (std::exception& ex) {
// This block allows us to investigate the exception using gdb
LOG(ERROR) << "Got an exception while setting up the server: " << ex.what();
handleSetupFailure();
throw;
} catch (...) {
handleSetupFailure();
throw;
}
}
void ThriftServer::setup() {
DCHECK_NOTNULL(cpp2Pfac_.get());
DCHECK_GT(nWorkers_, 0);
uint32_t threadsStarted = 0;
bool eventBaseAttached = false;
// Make sure EBM exists if we haven't set one explicitly
getEventBaseManager();
// Initialize event base for this thread, ensure event_init() is called
serveEventBase_ = eventBaseManager_->getEventBase();
// Print some libevent stats
LOG(INFO) << "libevent " <<
TEventBase::getLibeventVersion() << " method " <<
TEventBase::getLibeventMethod();
try {
// We check for write success so we don't need or want SIGPIPEs.
signal(SIGPIPE, SIG_IGN);
if (!observer_ && apache::thrift::observerFactory_) {
observer_ = apache::thrift::observerFactory_->getObserver();
}
// bind to the socket
if (socket_ == nullptr) {
socket_.reset(new TAsyncServerSocket());
socket_->setShutdownSocketSet(shutdownSocketSet_.get());
if (port_ != -1) {
socket_->bind(port_);
} else {
DCHECK(address_.isInitialized());
socket_->bind(address_);
}
}
socket_->listen(listenBacklog_);
socket_->setMaxNumMessagesInQueue(maxNumMsgsInQueue_);
socket_->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
// We always need a threadmanager for cpp2.
if (!threadFactory_) {
setThreadFactory(std::shared_ptr<ThreadFactory>(
new PosixThreadFactory));
}
if (!threadManager_) {
std::shared_ptr<apache::thrift::concurrency::ThreadManager>
threadManager(PriorityThreadManager::newPriorityThreadManager(
nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_,
true /*stats*/));
threadManager->enableCodel(getEnableCodel());
threadManager->start();
setThreadManager(threadManager);
}
threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
EventTask* task = dynamic_cast<EventTask*>(r.get());
if (task) {
task->expired();
}
});
threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
auto observer = getObserver();
if (observer) {
observer->queueTimeout();
}
});
auto b = std::make_shared<boost::barrier>(nWorkers_ + 1);
// Create the worker threads.
workers_.reserve(nWorkers_);
for (uint32_t n = 0; n < nWorkers_; ++n) {
addWorker();
workers_[n].worker->getEventBase()->runInLoop([b](){
b->wait();
});
}
// Update address_ with the address that we are actually bound to.
// (This is needed if we were supplied a pre-bound socket, or if address_'s
// port was set to 0, so an ephemeral port was chosen by the kernel.)
socket_->getAddress(&address_);
// Notify handler of the preServe event
if (eventHandler_ != nullptr) {
eventHandler_->preServe(&address_);
}
for (auto& worker: workers_) {
worker.thread->start();
++threadsStarted;
worker.thread->setName(folly::to<std::string>("Cpp2Worker",
threadsStarted));
}
// Wait for all workers to start
b->wait();
socket_->attachEventBase(eventBaseManager_->getEventBase());
eventBaseAttached = true;
socket_->startAccepting();
} catch (...) {
// XXX: Cpp2Worker::acceptStopped() calls
// eventBase_.terminateLoopSoon(). Normally this stops the
// worker from processing more work and stops the event loop.
// However, if startConsuming() and eventBase_.loop() haven't
// run yet this won't do the right thing. The worker thread
// will still continue to call startConsuming() later, and
// will then start the event loop.
for (uint32_t i = 0; i < threadsStarted; ++i) {
workers_[i].worker->acceptStopped();
workers_[i].thread->join();
}
workers_.clear();
if (socket_) {
if (eventBaseAttached) {
socket_->detachEventBase();
}
socket_.reset();
}
throw;
}
}
void ThriftServer::setup() {
DCHECK_NOTNULL(cpp2Pfac_.get());
DCHECK_GT(nWorkers_, 0);
uint32_t threadsStarted = 0;
// Initialize event base for this thread, ensure event_init() is called
serveEventBase_ = eventBaseManager_->getEventBase();
// Print some libevent stats
VLOG(1) << "libevent " <<
folly::EventBase::getLibeventVersion() << " method " <<
folly::EventBase::getLibeventMethod();
try {
// We check for write success so we don't need or want SIGPIPEs.
signal(SIGPIPE, SIG_IGN);
if (!observer_ && apache::thrift::observerFactory_) {
observer_ = apache::thrift::observerFactory_->getObserver();
}
// We always need a threadmanager for cpp2.
if (!threadFactory_) {
setThreadFactory(
std::make_shared<apache::thrift::concurrency::PosixThreadFactory>(
apache::thrift::concurrency::PosixThreadFactory::kDefaultPolicy,
apache::thrift::concurrency::PosixThreadFactory::kDefaultPriority,
threadStackSizeMB_
)
);
}
if (saslPolicy_ == "required" || saslPolicy_ == "permitted") {
if (!saslThreadManager_) {
auto numThreads = nSaslPoolThreads_ > 0
? nSaslPoolThreads_
: (nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_);
saslThreadManager_ = ThreadManager::newSimpleThreadManager(
numThreads,
0, /* pendingTaskCountMax -- no limit */
false, /* enableTaskStats */
0 /* maxQueueLen -- large default */);
saslThreadManager_->setNamePrefix("thrift-sasl");
saslThreadManager_->threadFactory(threadFactory_);
saslThreadManager_->start();
}
auto saslThreadManager = saslThreadManager_;
if (getSaslServerFactory()) {
// If the factory is already set, don't override it with the default
} else if (FLAGS_pin_service_identity &&
!FLAGS_service_identity.empty()) {
// If pin_service_identity flag is set and service_identity is specified
// force the server use the corresponding principal from keytab.
char hostname[256];
if (gethostname(hostname, 255)) {
LOG(FATAL) << "Failed getting hostname";
}
setSaslServerFactory([=] (folly::EventBase* evb) {
auto saslServer = std::unique_ptr<SaslServer>(
new GssSaslServer(evb, saslThreadManager));
saslServer->setServiceIdentity(
FLAGS_service_identity + "/" + hostname);
return saslServer;
});
} else {
// Allow the server to accept anything in the keytab.
setSaslServerFactory([=] (folly::EventBase* evb) {
return std::unique_ptr<SaslServer>(
new GssSaslServer(evb, saslThreadManager));
});
}
}
if (!threadManager_) {
int numThreads = nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_;
std::shared_ptr<apache::thrift::concurrency::ThreadManager>
threadManager(PriorityThreadManager::newPriorityThreadManager(
numThreads,
true /*stats*/,
getMaxRequests() + numThreads /*maxQueueLen*/));
threadManager->enableCodel(getEnableCodel());
if (!poolThreadName_.empty()) {
threadManager->setNamePrefix(poolThreadName_);
}
threadManager->start();
setThreadManager(threadManager);
}
threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
EventTask* task = dynamic_cast<EventTask*>(r.get());
if (task) {
task->expired();
}
});
threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
auto observer = getObserver();
if (observer) {
observer->queueTimeout();
}
});
if (!serverChannel_) {
ServerBootstrap::socketConfig.acceptBacklog = listenBacklog_;
// Resize the IO pool
ioThreadPool_->setNumThreads(nWorkers_);
ServerBootstrap::childHandler(
acceptorFactory_ ? acceptorFactory_
: std::make_shared<ThriftAcceptorFactory>(this));
{
std::lock_guard<std::mutex> lock(ioGroupMutex_);
ServerBootstrap::group(acceptPool_, ioThreadPool_);
}
if (socket_) {
ServerBootstrap::bind(std::move(socket_));
} else if (port_ != -1) {
ServerBootstrap::bind(port_);
} else {
ServerBootstrap::bind(address_);
}
// Update address_ with the address that we are actually bound to.
// (This is needed if we were supplied a pre-bound socket, or if
// address_'s port was set to 0, so an ephemeral port was chosen by
// the kernel.)
ServerBootstrap::getSockets()[0]->getAddress(&address_);
for (auto& socket : getSockets()) {
socket->setShutdownSocketSet(shutdownSocketSet_.get());
socket->setMaxNumMessagesInQueue(maxNumPendingConnectionsPerWorker_);
socket->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
}
// Notify handler of the preServe event
if (eventHandler_ != nullptr) {
eventHandler_->preServe(&address_);
}
} else {
CHECK(configMutable());
duplexWorker_ = folly::make_unique<Cpp2Worker>(this, serverChannel_);
// we don't control the EventBase for the duplexWorker, so when we shut
// it down, we need to ensure there's no delay
duplexWorker_->setGracefulShutdownTimeout(std::chrono::milliseconds(0));
}
// Do not allow setters to be called past this point until the IO worker
// threads have been joined in stopWorkers().
configMutable_ = false;
} catch (std::exception& ex) {
// This block allows us to investigate the exception using gdb
LOG(ERROR) << "Got an exception while setting up the server: "
<< ex.what();
handleSetupFailure();
throw;
} catch (...) {
handleSetupFailure();
throw;
}
}
void ThriftServer::setup() {
DCHECK_NOTNULL(cpp2Pfac_.get());
DCHECK_GT(nWorkers_, 0);
uint32_t threadsStarted = 0;
bool eventBaseAttached = false;
// Make sure EBM exists if we haven't set one explicitly
getEventBaseManager();
// Initialize event base for this thread, ensure event_init() is called
serveEventBase_ = eventBaseManager_->getEventBase();
// Print some libevent stats
LOG(INFO) << "libevent " <<
TEventBase::getLibeventVersion() << " method " <<
TEventBase::getLibeventMethod();
try {
// We check for write success so we don't need or want SIGPIPEs.
signal(SIGPIPE, SIG_IGN);
if (!observer_ && apache::thrift::observerFactory_) {
observer_ = apache::thrift::observerFactory_->getObserver();
}
// bind to the socket
if (!serverChannel_) {
if (socket_ == nullptr) {
socket_.reset(new TAsyncServerSocket());
socket_->setShutdownSocketSet(shutdownSocketSet_.get());
if (port_ != -1) {
socket_->bind(port_);
} else {
DCHECK(address_.isInitialized());
socket_->bind(address_);
}
}
socket_->listen(listenBacklog_);
socket_->setMaxNumMessagesInQueue(maxNumMsgsInQueue_);
socket_->setAcceptRateAdjustSpeed(acceptRateAdjustSpeed_);
}
// We always need a threadmanager for cpp2.
if (!threadFactory_) {
setThreadFactory(
std::make_shared<apache::thrift::concurrency::NumaThreadFactory>());
}
if (FLAGS_sasl_policy == "required" || FLAGS_sasl_policy == "permitted") {
if (!saslThreadManager_) {
saslThreadManager_ = ThreadManager::newSimpleThreadManager(
nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_, /* count */
0, /* pendingTaskCountMax -- no limit */
false, /* enableTaskStats */
0 /* maxQueueLen -- large default */);
saslThreadManager_->setNamePrefix("thrift-sasl");
saslThreadManager_->threadFactory(threadFactory_);
saslThreadManager_->start();
}
auto saslThreadManager = saslThreadManager_;
if (getSaslServerFactory()) {
// If the factory is already set, don't override it with the default
} else if (FLAGS_kerberos_service_name.empty()) {
// If the service name is not specified, not need to pin the principal.
// Allow the server to accept anything in the keytab.
setSaslServerFactory([=] (TEventBase* evb) {
return std::unique_ptr<SaslServer>(
new GssSaslServer(evb, saslThreadManager));
});
} else {
char hostname[256];
if (gethostname(hostname, 255)) {
LOG(FATAL) << "Failed getting hostname";
}
setSaslServerFactory([=] (TEventBase* evb) {
auto saslServer = std::unique_ptr<SaslServer>(
new GssSaslServer(evb, saslThreadManager));
saslServer->setServiceIdentity(
FLAGS_kerberos_service_name + "/" + hostname);
return std::move(saslServer);
});
}
}
if (!threadManager_) {
std::shared_ptr<apache::thrift::concurrency::ThreadManager>
threadManager(new apache::thrift::concurrency::NumaThreadManager(
nPoolThreads_ > 0 ? nPoolThreads_ : nWorkers_,
true /*stats*/,
getMaxRequests() /*maxQueueLen*/));
threadManager->enableCodel(getEnableCodel());
if (!poolThreadName_.empty()) {
threadManager->setNamePrefix(poolThreadName_);
}
threadManager->start();
setThreadManager(threadManager);
}
threadManager_->setExpireCallback([&](std::shared_ptr<Runnable> r) {
EventTask* task = dynamic_cast<EventTask*>(r.get());
if (task) {
task->expired();
}
});
threadManager_->setCodelCallback([&](std::shared_ptr<Runnable> r) {
auto observer = getObserver();
if (observer) {
observer->queueTimeout();
}
});
if (!serverChannel_) {
// regular server
auto b = std::make_shared<boost::barrier>(nWorkers_ + 1);
// Create the worker threads.
workers_.reserve(nWorkers_);
for (uint32_t n = 0; n < nWorkers_; ++n) {
addWorker();
workers_[n].worker->getEventBase()->runInLoop([b](){
b->wait();
});
}
// Update address_ with the address that we are actually bound to.
// (This is needed if we were supplied a pre-bound socket, or if
// address_'s port was set to 0, so an ephemeral port was chosen by
// the kernel.)
if (socket_) {
socket_->getAddress(&address_);
}
// Notify handler of the preServe event
if (eventHandler_ != nullptr) {
eventHandler_->preServe(&address_);
}
for (auto& worker: workers_) {
worker.thread->start();
++threadsStarted;
worker.thread->setName(folly::to<std::string>(cpp2WorkerThreadName_,
threadsStarted));
}
// Wait for all workers to start
b->wait();
if (socket_) {
socket_->attachEventBase(eventBaseManager_->getEventBase());
}
eventBaseAttached = true;
if (socket_) {
socket_->startAccepting();
}
} else {
// duplex server
// Create the Cpp2Worker
DCHECK(workers_.empty());
WorkerInfo info;
uint32_t workerID = 0;
info.worker.reset(new Cpp2Worker(this, workerID, serverChannel_));
// no thread, use current one (shared with client)
info.thread = nullptr;
workers_.push_back(info);
}
} catch (...) {
// XXX: Cpp2Worker::acceptStopped() calls
// eventBase_.terminateLoopSoon(). Normally this stops the
// worker from processing more work and stops the event loop.
// However, if startConsuming() and eventBase_.loop() haven't
// run yet this won't do the right thing. The worker thread
// will still continue to call startConsuming() later, and
// will then start the event loop.
if (!serverChannel_) {
for (uint32_t i = 0; i < threadsStarted; ++i) {
workers_[i].worker->acceptStopped();
workers_[i].thread->join();
}
}
workers_.clear();
if (socket_) {
if (eventBaseAttached) {
socket_->detachEventBase();
}
socket_.reset();
}
// avoid crash on stop()
serveEventBase_ = nullptr;
throw;
}
}