/** * Takes a socket created by listenSocket() and sets various options on it * to prepare for use in the server. */ void TNonblockingServer::listenSocket(THRIFT_SOCKET s) { // Set socket to nonblocking mode int flags; if ((flags = THRIFT_FCNTL(s, THRIFT_F_GETFL, 0)) < 0 || THRIFT_FCNTL(s, THRIFT_F_SETFL, flags | THRIFT_O_NONBLOCK) < 0) { ::THRIFT_CLOSESOCKET(s); throw TException("TNonblockingServer::serve() THRIFT_O_NONBLOCK"); } int one = 1; struct linger ling = {0, 0}; // Keepalive to ensure full result flushing setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, const_cast_sockopt(&one), sizeof(one)); // Turn linger off to avoid hung sockets setsockopt(s, SOL_SOCKET, SO_LINGER, const_cast_sockopt(&ling), sizeof(ling)); // Set TCP nodelay if available, MAC OS X Hack // See http://lists.danga.com/pipermail/memcached/2005-March/001240.html #ifndef TCP_NOPUSH setsockopt(s, IPPROTO_TCP, TCP_NODELAY, const_cast_sockopt(&one), sizeof(one)); #endif #ifdef TCP_LOW_MIN_RTO if (TSocket::getUseLowMinRto()) { setsockopt(s, IPPROTO_TCP, TCP_LOW_MIN_RTO, const_cast_sockopt(&one), sizeof(one)); } #endif if (listen(s, LISTEN_BACKLOG) == -1) { ::THRIFT_CLOSESOCKET(s); throw TException("TNonblockingServer::serve() listen"); } // Cool, this socket is good to go, set it as the serverSocket_ serverSocket_ = s; if (!port_) { struct sockaddr_storage addr; socklen_t size = sizeof(addr); if (!getsockname(serverSocket_, reinterpret_cast<sockaddr*>(&addr), &size)) { if (addr.ss_family == AF_INET6) { const struct sockaddr_in6* sin = reinterpret_cast<const struct sockaddr_in6*>(&addr); listenPort_ = ntohs(sin->sin6_port); } else { const struct sockaddr_in* sin = reinterpret_cast<const struct sockaddr_in*>(&addr); listenPort_ = ntohs(sin->sin_port); } } else { GlobalOutput.perror("TNonblocking: failed to get listen port: ", THRIFT_GET_SOCKET_ERROR); } } }
bool TNonblockingIOThread::notify(TNonblockingServer::TConnection* conn) { THRIFT_SOCKET fd = getNotificationSendFD(); if (fd < 0) { return false; } const int kSize = sizeof(conn); if (send(fd, const_cast_sockopt(&conn), kSize, 0) != kSize) { return false; } return true; }
/** * Takes a socket created by listenSocket() and sets various options on it * to prepare for use in the server. */ void TNonblockingServer::listenSocket(THRIFT_SOCKET s) { // Set socket to nonblocking mode int flags; if ((flags = THRIFT_FCNTL(s, THRIFT_F_GETFL, 0)) < 0 || THRIFT_FCNTL(s, THRIFT_F_SETFL, flags | THRIFT_O_NONBLOCK) < 0) { ::THRIFT_CLOSESOCKET(s); throw TException("TNonblockingServer::serve() THRIFT_O_NONBLOCK"); } int one = 1; struct linger ling = {0, 0}; // Keepalive to ensure full result flushing setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, const_cast_sockopt(&one), sizeof(one)); // Turn linger off to avoid hung sockets setsockopt(s, SOL_SOCKET, SO_LINGER, const_cast_sockopt(&ling), sizeof(ling)); // Set TCP nodelay if available, MAC OS X Hack // See http://lists.danga.com/pipermail/memcached/2005-March/001240.html #ifndef TCP_NOPUSH setsockopt(s, IPPROTO_TCP, TCP_NODELAY, const_cast_sockopt(&one), sizeof(one)); #endif #ifdef TCP_LOW_MIN_RTO if (TSocket::getUseLowMinRto()) { setsockopt(s, IPPROTO_TCP, TCP_LOW_MIN_RTO, const_cast_sockopt(&one), sizeof(one)); } #endif if (listen(s, LISTEN_BACKLOG) == -1) { ::THRIFT_CLOSESOCKET(s); throw TException("TNonblockingServer::serve() listen"); } // Cool, this socket is good to go, set it as the serverSocket_ serverSocket_ = s; }
bool TNonblockingIOThread::notify(TNonblockingServer::TConnection* conn) { THRIFT_SOCKET fd = getNotificationSendFD(); if (fd < 0) { return false; } fd_set wfds, efds; int ret = -1; int kSize = sizeof(conn); const char* pos = (const char*)const_cast_sockopt(&conn); while (kSize > 0) { FD_ZERO(&wfds); FD_ZERO(&efds); FD_SET(fd, &wfds); FD_SET(fd, &efds); ret = select(fd + 1, NULL, &wfds, &efds, NULL); if (ret < 0) { return false; } else if (ret == 0) { continue; } if (FD_ISSET(fd, &efds)) { ::THRIFT_CLOSESOCKET(fd); return false; } if (FD_ISSET(fd, &wfds)) { ret = send(fd, pos, kSize, 0); if (ret < 0) { if (errno == EAGAIN) { continue; } ::THRIFT_CLOSESOCKET(fd); return false; } kSize -= ret; pos += ret; } } return true; }
/** * Creates a socket to listen on and binds it to the local port. */ void TNonblockingServer::createAndListenOnSocket() { THRIFT_SOCKET s; struct addrinfo hints, *res, *res0; int error; char port[sizeof("65536") + 1]; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; sprintf(port, "%d", port_); // Wildcard address error = getaddrinfo(NULL, port, &hints, &res0); if (error) { throw TException("TNonblockingServer::serve() getaddrinfo " + string(THRIFT_GAI_STRERROR(error))); } // Pick the ipv6 address first since ipv4 addresses can be mapped // into ipv6 space. for (res = res0; res; res = res->ai_next) { if (res->ai_family == AF_INET6 || res->ai_next == NULL) break; } // Create the server socket s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); if (s == -1) { freeaddrinfo(res0); throw TException("TNonblockingServer::serve() socket() -1"); } #ifdef IPV6_V6ONLY if (res->ai_family == AF_INET6) { int zero = 0; if (-1 == setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, const_cast_sockopt(&zero), sizeof(zero))) { GlobalOutput("TServerSocket::listen() IPV6_V6ONLY"); } } #endif // #ifdef IPV6_V6ONLY int one = 1; // Set THRIFT_NO_SOCKET_CACHING to avoid 2MSL delay on server restart setsockopt(s, SOL_SOCKET, THRIFT_NO_SOCKET_CACHING, const_cast_sockopt(&one), sizeof(one)); if (::bind(s, res->ai_addr, static_cast<int>(res->ai_addrlen)) == -1) { ::THRIFT_CLOSESOCKET(s); freeaddrinfo(res0); throw TTransportException(TTransportException::NOT_OPEN, "TNonblockingServer::serve() bind", THRIFT_GET_SOCKET_ERROR); } // Done with the addr info freeaddrinfo(res0); // Set up this file descriptor for listening listenSocket(s); }
void TNonblockingIOThread::createNotificationPipe() { if(evutil_socketpair(AF_LOCAL, SOCK_STREAM, 0, notificationPipeFDs_) == -1) { GlobalOutput.perror("TNonblockingServer::createNotificationPipe ", EVUTIL_SOCKET_ERROR()); throw TException("can't create notification pipe"); } if(evutil_make_socket_nonblocking(notificationPipeFDs_[0])<0 || evutil_make_socket_nonblocking(notificationPipeFDs_[1])<0) { ::close(notificationPipeFDs_[0]); ::close(notificationPipeFDs_[1]); throw TException("TNonblockingServer::createNotificationPipe() O_NONBLOCK"); } for (int i = 0; i < 2; ++i) { #if LIBEVENT_VERSION_NUMBER < 0x02000000 int flags; if ((flags = fcntl(notificationPipeFDs_[i], F_GETFD, 0)) < 0 || fcntl(notificationPipeFDs_[i], F_SETFD, flags | FD_CLOEXEC) < 0) { #else if (evutil_make_socket_closeonexec(notificationPipeFDs_[i]) < 0) { #endif ::close(notificationPipeFDs_[0]); ::close(notificationPipeFDs_[1]); throw TException("TNonblockingServer::createNotificationPipe() " "FD_CLOEXEC"); } } } /** * Register the core libevent events onto the proper base. */ void TNonblockingIOThread::registerEvents() { if (listenSocket_ >= 0) { // Register the server event event_set(&serverEvent_, listenSocket_, EV_READ | EV_PERSIST, TNonblockingIOThread::listenHandler, server_); event_base_set(eventBase_, &serverEvent_); // Add the event and start up the server if (-1 == event_add(&serverEvent_, 0)) { throw TException("TNonblockingServer::serve(): " "event_add() failed on server listen event"); } GlobalOutput.printf("TNonblocking: IO thread #%d registered for listen.", number_); } createNotificationPipe(); // Create an event to be notified when a task finishes event_set(¬ificationEvent_, getNotificationRecvFD(), EV_READ | EV_PERSIST, TNonblockingIOThread::notifyHandler, this); // Attach to the base event_base_set(eventBase_, ¬ificationEvent_); // Add the event and start up the server if (-1 == event_add(¬ificationEvent_, 0)) { throw TException("TNonblockingServer::serve(): " "event_add() failed on task-done notification event"); } GlobalOutput.printf("TNonblocking: IO thread #%d registered for notify.", number_); } bool TNonblockingIOThread::notify(TNonblockingServer::TConnection* conn) { int fd = getNotificationSendFD(); if (fd < 0) { return false; } const int kSize = sizeof(conn); if (send(fd, const_cast_sockopt(&conn), kSize, 0) != kSize) { return false; } return true; }