void TMultiplexingServer::run()
{
// Listen socket
quint16 port = Tf::app()->appSettings().value("ListenPort").toUInt();
int sock = TApplicationServerBase::nativeListen(QHostAddress::Any, port);
if (sock > 0) {
tSystemDebug("listen successfully. port:%d", port);
} else {
tSystemError("Failed to set socket descriptor: %d", sock);
TApplicationServerBase::nativeClose(sock);
return;
}
listenSocket = sock;
setSocketOption(listenSocket);
maxWorkers = Tf::app()->maxNumberOfServers(10);
tSystemDebug("MaxWorkers: %d", maxWorkers);
// Get send buffer size and recv buffer size
int res, sendBufSize, recvBufSize;
socklen_t optlen = sizeof(int);
res = getsockopt(listenSocket, SOL_SOCKET, SO_SNDBUF, &sendBufSize, &optlen);
if (res < 0)
tSystemDebug("SO_SNDBUF: %d", sendBufSize);
res = getsockopt(listenSocket, SOL_SOCKET, SO_RCVBUF, &recvBufSize, &optlen);
if (res < 0)
tSystemDebug("SO_RCVBUF: %d", recvBufSize);
const int MaxEvents = 128;
struct epoll_event events[MaxEvents];
sendBufSize *= 0.8;
char *sndbuffer = new char[sendBufSize];
char *rcvbuffer = new char[recvBufSize];
// Create epoll
epollFd = epoll_create(1);
if (epollFd < 0) {
tSystemError("Failed epoll_create()");
goto socket_error;
}
if (epollAdd(listenSocket, EPOLLIN) < 0) {
tSystemError("Failed epoll_ctl()");
goto epoll_error;
}
for (;;) {
// Get send-request
getSendRequest();
// Check pending requests
while (!pendingRequests.isEmpty() && (int)threadCounter < maxWorkers) {
int fd = pendingRequests.takeFirst();
THttpBuffer &recvbuf = recvBuffers[fd];
emitIncomingRequest(fd, recvbuf);
}
// Poll Sending/Receiving/Incoming
int timeout = ((int)threadCounter > 0) ? 1 : 100;
int nfd = tf_epoll_wait(epollFd, events, MaxEvents, timeout);
int err = errno;
if (nfd < 0) {
tSystemError("Failed epoll_wait() : errno:%d", err);
break;
}
for (int i = 0; i < nfd; ++i) {
if (events[i].data.fd == listenSocket) {
if (!pendingRequests.isEmpty())
continue;
// Incoming connection
struct sockaddr_storage addr;
socklen_t addrlen = sizeof(addr);
int clt = ::accept(events[i].data.fd, (sockaddr *)&addr, &addrlen);
if (clt < 0) {
tSystemWarn("Failed accept");
continue;
}
setNonBlocking(clt);
if (epollAdd(clt, EPOLLIN) == 0) {
THttpBuffer &recvbuf = recvBuffers[clt];
recvbuf.clear();
recvbuf.setClientAddress(QHostAddress((sockaddr *)&addr));
}
} else {
int cltfd = events[i].data.fd;
if ( (events[i].events & EPOLLIN) ) {
// Receive data
int len = ::recv(cltfd, rcvbuffer, recvBufSize, 0);
err = errno;
if (len > 0) {
// Read successfully
THttpBuffer &recvbuf = recvBuffers[cltfd];
recvbuf.write(rcvbuffer, len);
if (recvbuf.canReadHttpRequest()) {
// Incoming a request
if ((int)threadCounter >= maxWorkers) {
pendingRequests << cltfd;
} else {
emitIncomingRequest(cltfd, recvbuf);
}
}
} else {
if (len < 0 && err != ECONNRESET) {
tSystemError("Failed recv : errno:%d", err);
}
// Disconnect
epollClose(cltfd);
continue;
}
}
if ( (events[i].events & EPOLLOUT) ) {
// Send data
THttpSendBuffer *sendbuf = sendBuffers[cltfd].first();
if (!sendbuf) {
tSystemError("Not found send-buffer");
epollClose(cltfd);
continue;
}
int len = sendbuf->read(sndbuffer, sendBufSize);
int sentlen = ::send(cltfd, sndbuffer, len, 0);
err = errno;
TAccessLogger &logger = sendbuf->accessLogger();
if (sentlen <= 0) {
if (err != ECONNRESET) {
tSystemError("Failed send : errno:%d", err);
}
// Access log
logger.setResponseBytes(-1);
logger.write();
epollClose(cltfd);
continue;
} else {
logger.setResponseBytes(logger.responseBytes() + sentlen);
if (len > sentlen) {
tSystemDebug("sendbuf prepend: len:%d", len - sentlen);
sendbuf->prepend(sndbuffer + sentlen, len - sentlen);
}
if (sendbuf->atEnd()) {
logger.write(); // Writes access log
sendbuf->release();
QQueue<THttpSendBuffer*> &que = sendBuffers[cltfd];
delete que.dequeue(); // delete send-buffer obj
// Prepare recv
if (que.isEmpty())
epollModify(cltfd, EPOLLIN);
}
}
}
}
}
// Check stop flag
if (stopped) {
if (listenSocket > 0) {
// Close the listen-socket
epollClose(listenSocket);
listenSocket = 0;
}
if (!recvBuffers.isEmpty()) {
for (QMapIterator<int, THttpBuffer> it(recvBuffers); it.hasNext(); ) {
it.next();
epollClose(it.key());
}
}
if (recvBuffers.isEmpty() && sendBuffers.isEmpty()) {
break;
}
}
}
epoll_error:
TF_CLOSE(epollFd);
epollFd = 0;
socket_error:
if (listenSocket > 0)
TF_CLOSE(listenSocket);
listenSocket = 0;
delete sndbuffer;
delete rcvbuffer;
}
//#define DEBUG
int main() {
pthread_t schedulerThread;
epfd = epollCreate(100);
struct threadpool *pool;
if ((pool = threadpool_init(100)) == NULL ) {
printf("Error! Failed to create a thread pool struct.\n");
exit(EXIT_FAILURE);
}
int sockfd;
int newsockfd, newschedule;
sockfd = Create();
int schedulerfd = Create();
if (Bind(35000, sockfd) < 0) {
printf("error in bind");
exit(0);
}
if (Bind(35001, schedulerfd) < 0) {
printf("error in bind");
exit(0);
}
Listen(sockfd, 10);
Listen(schedulerfd, 10);
pthread_create(&schedulerThread, 0, (void*) &schedule, NULL);
int i = 0;
while (i < 100) {
incClient();
newsockfd = Accept(sockfd);
if (numClient() <= 100) {
Send(newsockfd, "ACCEPT", strlen("ACCEPT") + 1);
newschedule = Accept(schedulerfd);
fprintf(stderr, "Connection established with scheduler");
sockets newsock;
newsock.active = 1;
newsock.priority = 1;
newsock.schedulerSocket = newschedule;
newsock.workerSocket = newsockfd;
socketList = addToList(socketList, newsock);
#ifdef DEBUG
if(socketList==NULL) {
fprintf(stderr,"socketlist is null");
exit(0);
}
#endif
epollAdd(epfd, newschedule);
int ret = threadpool_add_task(pool, negotiator, (void*) &newsockfd,
1);
} else {
Send(newsockfd, "REJECT", strlen("REJECT") + 1);
close(newsockfd);
}
i++;
}
sleep(10);
Close(sockfd);
printf("shutting down");
return 0;
}
