OSTHREAD_FUNC TCPDatagramCommunication::serverThread(void *parm){ TCPDatagramCommunication *tdc = (TCPDatagramCommunication*) parm; int fdaccept; int res=0; TaskScheduler *ts; NewServer *ns, *epollptr; NewServer ServerEventFdNs; // epoll data for the ServerEventFd int workerno; SLauncher->initThreadContext("SERVER", 0); // allows this thread to send // messages to others ts = tgetTaskScheduler(); tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM, tdc); // *!* increase thread priority above normal? //WARNING_INIT(); // initializes warning task int i, n, epfd; struct epoll_event ev; struct epoll_event *epevents=0; eventfd_t eventdummy; epevents = new epoll_event[MAX_EPOLL_EVENTS]; ServerEventFdNs.fd = tdc->ServerEventFd; ServerEventFdNs.handlerid = -1; epfd = epoll_create1(0); assert(epfd != -1); ev.events = EPOLLIN; ev.data.ptr = (void*) &ServerEventFdNs; res = epoll_ctl(epfd, EPOLL_CTL_ADD, tdc->ServerEventFd, &ev); assert(res==0); UDPDest ud; while (!tdc->ForceEndThreads){ ts->runOnce(); n = epoll_wait(epfd, epevents, MAX_EPOLL_EVENTS, -1); for (i=0; i < n; ++i){ epollptr = (NewServer*) epevents[i].data.ptr; if (epollptr->fd == tdc->ServerEventFd){ // used just to wake us up eventfd_read(tdc->ServerEventFd, &eventdummy); while (!tdc->newServerQueue.empty()){ ns = tdc->newServerQueue.dequeue(); ev.events = EPOLLIN; ev.data.ptr = (void*) ns; res = epoll_ctl(epfd, EPOLL_CTL_ADD, ns->fd, &ev); assert(res==0); res = listen(ns->fd, SOMAXCONN); if (res == -1) { printf("listen() failed: %d\n", errno); continue; } } continue; } if (epevents[i].events & EPOLLIN){ // read available, new connection to accept ud.sockaddr_len = sizeof(sockaddr_in); fdaccept = accept(epollptr->fd, (sockaddr*) &ud.destaddr, &ud.sockaddr_len); if (fdaccept == -1){ int err = errno; if (err != 0){ printf("%016llx accept:socket_error %d from %08x\n", (long long) Time::now(), errno, *(int*)&ud.destaddr.sin_addr); } continue; } //printf("accept:connection from %08x\n", *(int*)&ud.destaddr.sin_addr); setnonblock(fdaccept); #ifdef DISABLE_NAGLE int value = 1; res = setsockopt(fdaccept, IPPROTO_TCP, TCP_NODELAY, (char*) &value, sizeof(int)); if (res) printf("setsockopt on TCP_NODELAY of accept socket: error %d\n", errno); #endif // determines which worker will handle this fd workerno = tdc->ClientCount++; //printf("Fd %d going to worker %d\n", fdaccept, workerno); tdc->startReceiving(ud.getIPPort(), fdaccept, epollptr->handlerid, workerno); } // if if (epevents[i].events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)){ // problem with fd if (epollptr->fd != tdc->ServerEventFd){ printf("Problem with fd, closing it\n"); close(epollptr->fd); delete epollptr; } } // if } // for } delete [] epevents; return 0; }
OSTHREAD_FUNC TCPDatagramCommunication::workerThread(void *parm){ TCPDatagramCommunication *tdc = (TCPDatagramCommunication *) parm; TaskScheduler *ts = tgetTaskScheduler(); int epfd = epoll_create1(0); assert(epfd != -1); eventfd_t eventdummy; int myworkerno = gContext.indexWithinClass(TCLASS_WORKER, tgetThreadNo()); tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM, tdc); tsetSharedSpace(THREADCONTEXT_SPACE_TCPDATAGRAM_WORKER, (void*)(long long) epfd); ts->assignImmediateFunc(IMMEDIATEFUNC_ADDIPPORTFD, immediateFuncAddIPPortFd); ts->assignImmediateFunc(IMMEDIATEFUNC_SEND, immediateFuncSend); tdc->startupWorkerThread(); // invokes startup code int i, n, res; struct epoll_event *epevents=0; TCPStreamState *tss=0; TCPStreamState sleepeventtss; epevents = new epoll_event[MAX_EPOLL_EVENTS]; int nread=0; int sleepeventfd = ts->getSleepEventFd(); struct epoll_event ev; sleepeventtss.fd = sleepeventfd; ev.events = EPOLLIN; ev.data.ptr = (void*) &sleepeventtss; res = epoll_ctl(epfd, EPOLL_CTL_ADD, sleepeventfd, &ev); assert(res==0); int something; int timeout=0; tdc->workerInitSync.signal(); // indicate that we have initialized while (!tdc->ForceEndThreads){ something = ts->runOnce(); // run event schedule loop once // this will handle immediate functions to add new // things to the epoll set: // after adding, must receive anything that already exists // go through pendingsends and send anything for which we did not get // EAGAIN before if (!tdc->PendingSendsBeforeEpoll[myworkerno].empty()){ SetNode<TCPStreamStatePtr> *it; TCPStreamState *tssptr; for (it = tdc->PendingSendsBeforeEpoll[myworkerno].getFirst(); it != tdc->PendingSendsBeforeEpoll[myworkerno].getLast(); it = tdc->PendingSendsBeforeEpoll[myworkerno].getNext(it)){ tssptr = it->key.tssptr; assert(!tssptr->sendeagain); tdc->sendTss(tssptr); } tdc->PendingSendsBeforeEpoll[myworkerno].clear(); } if (!something){ // start sleep cycle ts->setAsleep(1); timeout = ts->findSleepTimeout(); //printf("Going to sleep for %d\n", timeout); } else timeout = 0; n = epoll_wait(epfd, epevents, MAX_EPOLL_EVENTS, timeout); if (!something) ts->setAsleep(0); for (i=0; i < n; ++i){ tss = (TCPStreamState*) epevents[i].data.ptr; if (tss->fd == sleepeventfd){ // used just to wake us up //printf("Woken from sleep\n"); eventfd_read(sleepeventfd, &eventdummy); continue; } if (epevents[i].events & EPOLLIN){ // read available //if (ts->checkSendQueuesAlmostFull()){ // **!** if internal send queues are almost full, do not receive // TCP packets // do something here //} while (1){ nread = read(tss->fd, tss->rstate.Ptr, tss->rstate.Buflen - tss->rstate.Filled); if (nread < 0){ if (errno == EAGAIN || errno == EWOULDBLOCK) break; } else if (nread == 0) break; else { // update the state of this stream (and if entire message received, // invoke handler) tdc->updateState(tss->handlerid, tss->rstate, tss->ipport, nread); } } } if (epevents[i].events & EPOLLOUT){ // write available tdc->sendTss(tss); } if (epevents[i].events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)){ // problem with fd close(tss->fd); } } } tdc->finishWorkerThread(); // invokes cleaning up code delete [] epevents; return 0; }