OSTHREAD_FUNC TCPDatagramCommunication::serverThread(void *parm){
TCPDatagramCommunication *tdc = (TCPDatagramCommunication*) parm;
int fdaccept;
int res=0;
TaskScheduler *ts;
NewServer *ns, *epollptr;
NewServer ServerEventFdNs; // epoll data for the ServerEventFd
int workerno;
SLauncher->initThreadContext("SERVER", 0); // allows this thread to send
// messages to others
ts = tgetTaskScheduler();
tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM, tdc);
// *!* increase thread priority above normal?
//WARNING_INIT(); // initializes warning task
int i, n, epfd;
struct epoll_event ev;
struct epoll_event *epevents=0;
eventfd_t eventdummy;
epevents = new epoll_event[MAX_EPOLL_EVENTS];
ServerEventFdNs.fd = tdc->ServerEventFd;
ServerEventFdNs.handlerid = -1;
epfd = epoll_create1(0); assert(epfd != -1);
ev.events = EPOLLIN;
ev.data.ptr = (void*) &ServerEventFdNs;
res = epoll_ctl(epfd, EPOLL_CTL_ADD, tdc->ServerEventFd, &ev);
assert(res==0);
UDPDest ud;
while (!tdc->ForceEndThreads){
ts->runOnce();
n = epoll_wait(epfd, epevents, MAX_EPOLL_EVENTS, -1);
for (i=0; i < n; ++i){
epollptr = (NewServer*) epevents[i].data.ptr;
if (epollptr->fd == tdc->ServerEventFd){ // used just to wake us up
eventfd_read(tdc->ServerEventFd, &eventdummy);
while (!tdc->newServerQueue.empty()){
ns = tdc->newServerQueue.dequeue();
ev.events = EPOLLIN;
ev.data.ptr = (void*) ns;
res = epoll_ctl(epfd, EPOLL_CTL_ADD, ns->fd, &ev); assert(res==0);
res = listen(ns->fd, SOMAXCONN);
if (res == -1) {
printf("listen() failed: %d\n", errno);
continue;
}
}
continue;
}
if (epevents[i].events & EPOLLIN){
// read available, new connection to accept
ud.sockaddr_len = sizeof(sockaddr_in);
fdaccept = accept(epollptr->fd, (sockaddr*) &ud.destaddr,
&ud.sockaddr_len);
if (fdaccept == -1){
int err = errno;
if (err != 0){
printf("%016llx accept:socket_error %d from %08x\n",
(long long) Time::now(), errno, *(int*)&ud.destaddr.sin_addr);
}
continue;
}
//printf("accept:connection from %08x\n", *(int*)&ud.destaddr.sin_addr);
setnonblock(fdaccept);
#ifdef DISABLE_NAGLE
int value = 1;
res = setsockopt(fdaccept, IPPROTO_TCP, TCP_NODELAY, (char*) &value,
sizeof(int));
if (res)
printf("setsockopt on TCP_NODELAY of accept socket: error %d\n",
errno);
#endif
// determines which worker will handle this fd
workerno = tdc->ClientCount++;
//printf("Fd %d going to worker %d\n", fdaccept, workerno);
tdc->startReceiving(ud.getIPPort(), fdaccept, epollptr->handlerid,
workerno);
} // if
if (epevents[i].events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)){
// problem with fd
if (epollptr->fd != tdc->ServerEventFd){
printf("Problem with fd, closing it\n");
close(epollptr->fd);
delete epollptr;
}
} // if
} // for
}
delete [] epevents;
return 0;
}
OSTHREAD_FUNC TCPDatagramCommunication::workerThread(void *parm){
TCPDatagramCommunication *tdc = (TCPDatagramCommunication *) parm;
TaskScheduler *ts = tgetTaskScheduler();
int epfd = epoll_create1(0); assert(epfd != -1);
eventfd_t eventdummy;
int myworkerno = gContext.indexWithinClass(TCLASS_WORKER, tgetThreadNo());
tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM, tdc);
tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM_WORKER,
(void*)(long long) epfd);
ts->assignImmediateFunc(IMMEDIATEFUNC_ADDIPPORTFD, immediateFuncAddIPPortFd);
ts->assignImmediateFunc(IMMEDIATEFUNC_SEND, immediateFuncSend);
tdc->startupWorkerThread(); // invokes startup code
int i, n, res;
struct epoll_event *epevents=0;
TCPStreamState *tss=0;
TCPStreamState sleepeventtss;
epevents = new epoll_event[MAX_EPOLL_EVENTS];
int nread=0;
int sleepeventfd = ts->getSleepEventFd();
struct epoll_event ev;
sleepeventtss.fd = sleepeventfd;
ev.events = EPOLLIN;
ev.data.ptr = (void*) &sleepeventtss;
res = epoll_ctl(epfd, EPOLL_CTL_ADD, sleepeventfd, &ev); assert(res==0);
int something;
int timeout=0;
tdc->workerInitSync.signal(); // indicate that we have initialized
while (!tdc->ForceEndThreads){
something = ts->runOnce(); // run event schedule loop once
// this will handle immediate functions to add new
// things to the epoll set:
// after adding, must receive anything that already exists
// go through pendingsends and send anything for which we did not get
// EAGAIN before
if (!tdc->PendingSendsBeforeEpoll[myworkerno].empty()){
SetNode<TCPStreamStatePtr> *it;
TCPStreamState *tssptr;
for (it = tdc->PendingSendsBeforeEpoll[myworkerno].getFirst();
it != tdc->PendingSendsBeforeEpoll[myworkerno].getLast();
it = tdc->PendingSendsBeforeEpoll[myworkerno].getNext(it)){
tssptr = it->key.tssptr;
assert(!tssptr->sendeagain);
tdc->sendTss(tssptr);
}
tdc->PendingSendsBeforeEpoll[myworkerno].clear();
}
if (!something){ // start sleep cycle
ts->setAsleep(1);
timeout = ts->findSleepTimeout();
//printf("Going to sleep for %d\n", timeout);
} else timeout = 0;
n = epoll_wait(epfd, epevents, MAX_EPOLL_EVENTS, timeout);
if (!something) ts->setAsleep(0);
for (i=0; i < n; ++i){
tss = (TCPStreamState*) epevents[i].data.ptr;
if (tss->fd == sleepeventfd){ // used just to wake us up
//printf("Woken from sleep\n");
eventfd_read(sleepeventfd, &eventdummy);
continue;
}
if (epevents[i].events & EPOLLIN){ // read available
//if (ts->checkSendQueuesAlmostFull()){
// **!** if internal send queues are almost full, do not receive
// TCP packets
// do something here
//}
while (1){
nread = read(tss->fd, tss->rstate.Ptr,
tss->rstate.Buflen - tss->rstate.Filled);
if (nread < 0){
if (errno == EAGAIN || errno == EWOULDBLOCK) break;
} else if (nread == 0) break;
else {
// update the state of this stream (and if entire message received,
// invoke handler)
tdc->updateState(tss->handlerid, tss->rstate, tss->ipport, nread);
}
}
}
if (epevents[i].events & EPOLLOUT){ // write available
tdc->sendTss(tss);
}
if (epevents[i].events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)){
// problem with fd
close(tss->fd);
}
}
}
tdc->finishWorkerThread(); // invokes cleaning up code
delete [] epevents;
return 0;
}
