IOManagerEPoll::IOManagerEPoll(int threads, bool useCaller) : Scheduler(threads, useCaller) { m_epfd = epoll_create(5000); MORDOR_LOG_LEVEL(g_log, m_epfd <= 0 ? Log::ERROR : Log::TRACE) << this << " epoll_create(5000): " << m_epfd; if (m_epfd <= 0) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_create"); int rc = pipe(m_tickleFds); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " pipe(): " << rc << " (" << errno << ")"; if (rc) { close(m_epfd); MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("pipe"); } MORDOR_ASSERT(m_tickleFds[0] > 0); MORDOR_ASSERT(m_tickleFds[1] > 0); epoll_event event; event.events = EPOLLIN; event.data.fd = m_tickleFds[0]; rc = epoll_ctl(m_epfd, EPOLL_CTL_ADD, m_tickleFds[0], &event); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " epoll_ctl(" << m_epfd << ", EPOLL_CTL_ADD, " << m_tickleFds[0] << ", EPOLLIN): " << rc << " (" << errno << ")"; if (rc) { close(m_tickleFds[0]); close(m_tickleFds[1]); close(m_epfd); MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_ctl"); } }
size_t HandleStream::write(const void *buffer, size_t length) { if (m_cancelWrite) MORDOR_THROW_EXCEPTION(OperationAbortedException()); SchedulerSwitcher switcher(m_ioManager ? NULL : m_scheduler); DWORD written; OVERLAPPED *overlapped = NULL; if (m_ioManager) { MORDOR_ASSERT(Scheduler::getThis()); m_ioManager->registerEvent(&m_writeEvent); overlapped = &m_writeEvent.overlapped; if (supportsSeek()) { overlapped->Offset = (DWORD)m_pos; overlapped->OffsetHigh = (DWORD)(m_pos >> 32); } } length = (std::min)(length, m_maxOpSize); BOOL ret = WriteFile(m_hFile, buffer, (DWORD)length, &written, overlapped); Log::Level level = Log::DEBUG; if (!ret) { if (m_ioManager && GetLastError() == ERROR_IO_PENDING) level = Log::TRACE; else level = Log::ERROR; } DWORD error = GetLastError(); MORDOR_LOG_LEVEL(g_log, level) << this << " WriteFile(" << m_hFile << ", " << length << "): " << ret << " - " << written << " (" << error << ")"; if (m_ioManager) { if (!ret && error != ERROR_IO_PENDING) { m_ioManager->unregisterEvent(&m_writeEvent); MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("WriteFile"); } if (m_skipCompletionPortOnSuccess && ret) m_ioManager->unregisterEvent(&m_writeEvent); else Scheduler::yieldTo(); DWORD error = pRtlNtStatusToDosError((NTSTATUS)m_writeEvent.overlapped.Internal); MORDOR_LOG_LEVEL(g_log, error ? Log::ERROR : Log::VERBOSE) << this << " WriteFile(" << m_hFile << ", " << length << "): " << m_writeEvent.overlapped.InternalHigh << " (" << error << ")"; if (error) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "WriteFile"); if (supportsSeek()) { m_pos = ((long long)overlapped->Offset | ((long long)overlapped->OffsetHigh << 32)) + m_writeEvent.overlapped.InternalHigh; } return m_writeEvent.overlapped.InternalHigh; } if (!ret) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "WriteFile"); return written; }
HandleStream::~HandleStream() { if (m_hFile != INVALID_HANDLE_VALUE && m_own) { SchedulerSwitcher switcher(m_scheduler); BOOL result = CloseHandle(m_hFile); MORDOR_LOG_LEVEL(g_log, result ? Log::VERBOSE : Log::ERROR) << this << " CloseHandle(" << m_hFile << "): " << result << " (" << GetLastError() << ")"; } }
void SSLStream::verifyPeerCertificate() { const long verifyResult = sslCallWithLock(std::bind(SSL_get_verify_result, m_ssl.get()), NULL); MORDOR_LOG_LEVEL(g_log, verifyResult ? Log::WARNING : Log::DBG) << this << " SSL_get_verify_result(" << m_ssl.get() << "): " << verifyResult; if (verifyResult != X509_V_OK) MORDOR_THROW_EXCEPTION(CertificateVerificationException(verifyResult)); }
void HandleStream::flush(bool flushParent) { SchedulerSwitcher switcher(m_scheduler); BOOL ret = FlushFileBuffers(m_hFile); DWORD error = GetLastError(); MORDOR_LOG_LEVEL(g_log, ret ? Log::VERBOSE : Log::ERROR) << this << " FlushFileBuffers(" << m_hFile << "): " << ret << " (" << error << ")"; if (!ret) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "FlushFileBuffers"); }
void IOManagerEPoll::cancelEvent(int fd, Event events) { boost::mutex::scoped_lock lock(m_mutex); std::map<int, AsyncEvent>::iterator it = m_pendingEvents.find(fd); if (it == m_pendingEvents.end()) return; AsyncEvent &e = it->second; if ((events & EPOLLIN) && (e.event.events & EPOLLIN)) { if (e.m_dgIn) e.m_schedulerIn->schedule(e.m_dgIn); else e.m_schedulerIn->schedule(e.m_fiberIn); e.m_dgIn = NULL; e.m_fiberIn.reset(); } if ((events & EPOLLOUT) && (e.event.events & EPOLLOUT)) { if (e.m_dgOut) e.m_schedulerOut->schedule(e.m_dgOut); else e.m_schedulerOut->schedule(e.m_fiberOut); e.m_dgOut = NULL; e.m_fiberOut.reset(); } if ((events & EPOLLHUP) && (e.event.events & EPOLLHUP)) { if (e.m_dgClose) e.m_schedulerClose->schedule(e.m_dgClose); else e.m_schedulerClose->schedule(e.m_fiberClose); e.m_dgClose = NULL; e.m_fiberClose.reset(); } if ((events & EPOLLERR) && (e.event.events & EPOLLERR)) { if (e.m_dgError) e.m_schedulerError->schedule(e.m_dgError); else e.m_schedulerError->schedule(e.m_fiberError); e.m_dgError = NULL; e.m_fiberError.reset(); } e.event.events &= ~events; int op = e.event.events ? EPOLL_CTL_MOD : EPOLL_CTL_DEL; int rc = epoll_ctl(m_epfd, op, fd, &e.event); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " epoll_ctl(" << m_epfd << ", " << (epoll_ctl_op_t)op << ", " << fd << ", " << (epoll_events_t)e.event.events << "): " << rc << " (" << errno << ")"; if (op == EPOLL_CTL_DEL) m_pendingEvents.erase(it); if (rc) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_ctl"); }
void HandleStream::truncate(long long size) { SchedulerSwitcher switcher(m_scheduler); long long pos = seek(0, CURRENT); seek(size, BEGIN); BOOL ret = SetEndOfFile(m_hFile); DWORD error = GetLastError(); MORDOR_LOG_LEVEL(g_log, ret ? Log::VERBOSE : Log::ERROR) << this << " SetEndOfFile(" << m_hFile << "): " << ret << " (" << error << ")"; seek(pos, BEGIN); if (!ret) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "SetEndOfFile"); }
void HandleStream::close(CloseType type) { MORDOR_ASSERT(type == BOTH); if (m_hFile != INVALID_HANDLE_VALUE && m_own) { SchedulerSwitcher switcher(m_scheduler); BOOL result = CloseHandle(m_hFile); DWORD error = GetLastError(); MORDOR_LOG_LEVEL(g_log, result ? Log::VERBOSE : Log::ERROR) << this << " CloseHandle(" << m_hFile << "): " << result << " (" << error << ")"; if (!result) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "CloseHandle"); m_hFile = INVALID_HANDLE_VALUE; } }
long long HandleStream::seek(long long offset, Anchor anchor) { SchedulerSwitcher switcher(m_ioManager ? NULL : m_scheduler); if (m_ioManager) { if (supportsSeek()) { switch (anchor) { case BEGIN: if (offset < 0) { MORDOR_THROW_EXCEPTION(std::invalid_argument("resulting offset is negative")); } return m_pos = offset; case CURRENT: if (m_pos + offset < 0) { MORDOR_THROW_EXCEPTION(std::invalid_argument("resulting offset is negative")); } return m_pos += offset; case END: { long long end = size(); if (end + offset < 0) { MORDOR_THROW_EXCEPTION(std::invalid_argument("resulting offset is negative")); } return m_pos = end + offset; } default: MORDOR_ASSERT(false); } } else { MORDOR_ASSERT(false); } } long long pos; BOOL ret = SetFilePointerEx(m_hFile, *(LARGE_INTEGER*)&offset, (LARGE_INTEGER*)&pos, (DWORD)anchor); DWORD error = GetLastError(); MORDOR_LOG_LEVEL(g_log, ret ? Log::VERBOSE : Log::ERROR) << this << " SetFilePointerEx(" << m_hFile << ", " << offset << ", " << pos << ", " << anchor << "): " << ret << " (" << error << ")"; if (!ret) MORDOR_THROW_EXCEPTION_FROM_ERROR_API(error, "SetFilePointerEx"); return pos; }
void IOManagerEPoll::idle() { epoll_event events[64]; while (true) { unsigned long long nextTimeout; if (stopping(nextTimeout)) return; int rc = -1; errno = EINTR; while (rc < 0 && errno == EINTR) { int timeout = -1; if (nextTimeout != ~0ull) timeout = (int)(nextTimeout / 1000) + 1; rc = epoll_wait(m_epfd, events, 64, timeout); if (rc < 0 && errno == EINTR) nextTimeout = nextTimer(); } MORDOR_LOG_LEVEL(g_log, rc < 0 ? Log::ERROR : Log::VERBOSE) << this << " epoll_wait(" << m_epfd << "): " << rc << " (" << errno << ")"; if (rc < 0) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_wait"); std::vector<boost::function<void ()> > expired = processTimers(); schedule(expired.begin(), expired.end()); for(int i = 0; i < rc; ++i) { epoll_event &event = events[i]; if (event.data.fd == m_tickleFds[0]) { unsigned char dummy; int rc2 = read(m_tickleFds[0], &dummy, 1); MORDOR_VERIFY(rc2 == 1); MORDOR_LOG_VERBOSE(g_log) << this << " received tickle"; continue; } bool err = event.events & (EPOLLERR | EPOLLHUP); boost::mutex::scoped_lock lock(m_mutex); std::map<int, AsyncEvent>::iterator it = m_pendingEvents.find(event.data.fd); if (it == m_pendingEvents.end()) continue; AsyncEvent &e = it->second; MORDOR_LOG_TRACE(g_log) << " epoll_event {" << (epoll_events_t)event.events << ", " << event.data.fd << "}, registered for " << (epoll_events_t)e.event.events; if ((event.events & EPOLLERR) && (e.event.events & EPOLLERR)) { if (e.m_dgError) e.m_schedulerError->schedule(e.m_dgError); else e.m_schedulerError->schedule(e.m_fiberError); // Block other events from firing e.m_dgError = NULL; e.m_fiberError.reset(); e.m_dgIn = NULL; e.m_fiberIn.reset(); e.m_dgOut = NULL; e.m_fiberOut.reset(); event.events = 0; e.event.events = 0; } if ((event.events & EPOLLHUP) && (e.event.events & EPOLLHUP)) { if (e.m_dgClose) e.m_schedulerError->schedule(e.m_dgClose); else e.m_schedulerError->schedule(e.m_fiberClose); // Block write event from firing e.m_dgOut = NULL; e.m_fiberOut.reset(); e.m_dgClose = NULL; e.m_fiberClose.reset(); event.events &= EPOLLOUT; e.event.events &= EPOLLOUT; err = false; } if (((event.events & EPOLLIN) || err) && (e.event.events & EPOLLIN)) { if (e.m_dgIn) e.m_schedulerIn->schedule(e.m_dgIn); else e.m_schedulerIn->schedule(e.m_fiberIn); e.m_dgIn = NULL; e.m_fiberIn.reset(); event.events |= EPOLLIN; } if (((event.events & EPOLLOUT) || err) && (e.event.events & EPOLLOUT)) { if (e.m_dgOut) e.m_schedulerOut->schedule(e.m_dgOut); else e.m_schedulerOut->schedule(e.m_fiberOut); e.m_dgOut = NULL; e.m_fiberOut.reset(); event.events |= EPOLLOUT; } e.event.events &= ~event.events; int op = e.event.events == 0 ? EPOLL_CTL_DEL : EPOLL_CTL_MOD; int rc2 = epoll_ctl(m_epfd, op, event.data.fd, &e.event); MORDOR_LOG_LEVEL(g_log, rc2 ? Log::ERROR : Log::VERBOSE) << this << " epoll_ctl(" << m_epfd << ", " << (epoll_ctl_op_t)op << ", " << event.data.fd << ", " << (epoll_events_t)e.event.events << "): " << rc2 << " (" << errno << ")"; if (rc2) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_ctl"); if (op == EPOLL_CTL_DEL) m_pendingEvents.erase(it); } Fiber::yield(); } }
void IOManagerEPoll::registerEvent(int fd, Event events, boost::function<void ()> dg) { MORDOR_ASSERT(fd > 0); MORDOR_ASSERT(Scheduler::getThis()); MORDOR_ASSERT(Fiber::getThis()); int epollevents = (int)events & (EPOLLIN | EPOLLOUT | EPOLLHUP | EPOLLERR); MORDOR_ASSERT(epollevents != 0); boost::mutex::scoped_lock lock(m_mutex); int op; std::map<int, AsyncEvent>::iterator it = m_pendingEvents.find(fd); AsyncEvent *event; if (it == m_pendingEvents.end()) { op = EPOLL_CTL_ADD; event = &m_pendingEvents[fd]; event->event.data.fd = fd; event->event.events = epollevents; } else { op = EPOLL_CTL_MOD; event = &it->second; // OR == XOR means that none of the same bits were set MORDOR_ASSERT((event->event.events | epollevents) == (event->event.events ^ epollevents)); event->event.events |= epollevents; } if (epollevents & EPOLLIN) { event->m_schedulerIn = Scheduler::getThis(); if (dg) { event->m_dgIn = dg; event->m_fiberIn.reset(); } else { event->m_dgIn = NULL; event->m_fiberIn = Fiber::getThis(); } } if (epollevents & EPOLLOUT) { event->m_schedulerOut = Scheduler::getThis(); if (dg) { event->m_dgOut = dg; event->m_fiberOut.reset(); } else { event->m_dgOut = NULL; event->m_fiberOut = Fiber::getThis(); } } if (epollevents & EPOLLHUP) { event->m_schedulerClose = Scheduler::getThis(); if (dg) { event->m_dgClose = dg; event->m_fiberClose.reset(); } else { event->m_dgClose = NULL; event->m_fiberClose = Fiber::getThis(); } } if (epollevents & EPOLLERR) { event->m_schedulerError = Scheduler::getThis(); if (dg) { event->m_dgError = dg; event->m_fiberError.reset(); } else { event->m_dgError = NULL; event->m_fiberError = Fiber::getThis(); } } int rc = epoll_ctl(m_epfd, op, event->event.data.fd, &event->event); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " epoll_ctl(" << m_epfd << ", " << (epoll_ctl_op_t)op << ", " << event->event.data.fd << ", " << (epoll_events_t)event->event.events << "): " << rc << " (" << errno << ")"; if (rc) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_ctl"); }