void FiberMutex::lock() { MORDOR_ASSERT(Scheduler::getThis()); { boost::mutex::scoped_lock scopeLock(m_mutex); MORDOR_ASSERT(m_owner != Fiber::getThis()); MORDOR_ASSERT(std::find(m_waiters.begin(), m_waiters.end(), std::make_pair(Scheduler::getThis(), Fiber::getThis())) == m_waiters.end()); if (!m_owner) { m_owner = Fiber::getThis(); return; } m_waiters.push_back(std::make_pair(Scheduler::getThis(), Fiber::getThis())); } Scheduler::yieldTo(); #ifdef DEBUG boost::mutex::scoped_lock scopeLock(m_mutex); MORDOR_ASSERT(m_owner == Fiber::getThis()); MORDOR_ASSERT(std::find(m_waiters.begin(), m_waiters.end(), std::make_pair(Scheduler::getThis(), Fiber::getThis())) == m_waiters.end()); #endif }
IOManagerEPoll::IOManagerEPoll(int threads, bool useCaller) : Scheduler(threads, useCaller) { m_epfd = epoll_create(5000); MORDOR_LOG_LEVEL(g_log, m_epfd <= 0 ? Log::ERROR : Log::TRACE) << this << " epoll_create(5000): " << m_epfd; if (m_epfd <= 0) MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_create"); int rc = pipe(m_tickleFds); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " pipe(): " << rc << " (" << errno << ")"; if (rc) { close(m_epfd); MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("pipe"); } MORDOR_ASSERT(m_tickleFds[0] > 0); MORDOR_ASSERT(m_tickleFds[1] > 0); epoll_event event; event.events = EPOLLIN; event.data.fd = m_tickleFds[0]; rc = epoll_ctl(m_epfd, EPOLL_CTL_ADD, m_tickleFds[0], &event); MORDOR_LOG_LEVEL(g_log, rc ? Log::ERROR : Log::VERBOSE) << this << " epoll_ctl(" << m_epfd << ", EPOLL_CTL_ADD, " << m_tickleFds[0] << ", EPOLLIN): " << rc << " (" << errno << ")"; if (rc) { close(m_tickleFds[0]); close(m_tickleFds[1]); close(m_epfd); MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("epoll_ctl"); } }
size_t SSLStream::read(void *buffer, size_t length) { const int toRead = (int)std::min<size_t>(0x0fffffff, length); while (true) { unsigned long error = SSL_ERROR_NONE; const int result = sslCallWithLock(std::bind(SSL_read, m_ssl.get(), buffer, toRead), &error); if (result > 0) { return result; } MORDOR_LOG_DEBUG(g_log) << this << " SSL_read(" << m_ssl.get() << ", " << toRead << "): " << result << " (" << error << ")"; switch (error) { case SSL_ERROR_NONE: return result; case SSL_ERROR_ZERO_RETURN: // Received close_notify message MORDOR_ASSERT(result == 0); return 0; case SSL_ERROR_WANT_READ: wantRead(); continue; case SSL_ERROR_WANT_WRITE: case SSL_ERROR_WANT_CONNECT: case SSL_ERROR_WANT_ACCEPT: case SSL_ERROR_WANT_X509_LOOKUP: MORDOR_NOTREACHED(); case SSL_ERROR_SYSCALL: if (hasOpenSSLError()) { std::string message = getOpenSSLErrorMessage(); MORDOR_LOG_ERROR(g_log) << this << " SSL_read(" << m_ssl.get() << ", " << toRead << "): " << result << " (" << error << ", " << message << ")"; MORDOR_THROW_EXCEPTION(OpenSSLException(message)) // << boost::errinfo_api_function("SSL_read"); ; } MORDOR_LOG_WARNING(g_log) << this << " SSL_read(" << m_ssl.get() << ", " << toRead << "): " << result << " (" << error << ")"; if (result == 0) { return 0; } MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("SSL_read"); case SSL_ERROR_SSL: { MORDOR_ASSERT(hasOpenSSLError()); std::string message = getOpenSSLErrorMessage(); MORDOR_LOG_ERROR(g_log) << this << " SSL_read(" << m_ssl.get() << ", " << toRead << "): " << result << " (" << error << ", " << message << ")"; MORDOR_THROW_EXCEPTION(OpenSSLException(message)) // << boost::errinfo_api_function("SSL_read"); ; } default: MORDOR_NOTREACHED(); } } }
void Buffer::truncate(size_t length) { MORDOR_ASSERT(length <= readAvailable()); if (length == m_readAvailable) return; // Split any mixed read/write bufs if (m_writeIt != m_segments.end() && m_writeIt->readAvailable() != 0) { m_segments.insert(m_writeIt, Segment(m_writeIt->readBuffer())); m_writeIt->consume(m_writeIt->readAvailable()); } m_readAvailable = length; std::list<Segment>::iterator it; for (it = m_segments.begin(); it != m_segments.end() && length > 0; ++it) { Segment &segment = *it; if (length <= segment.readAvailable()) { segment.truncate(length); length = 0; ++it; break; } else { length -= segment.readAvailable(); } } MORDOR_ASSERT(length == 0); while (it != m_segments.end() && it->readAvailable() > 0) { MORDOR_ASSERT(it->writeAvailable() == 0); it = m_segments.erase(it); } invariant(); }
void Buffer::copyOut(void *buffer, size_t length, size_t pos) const { if (length == 0) return; MORDOR_ASSERT(length + pos <= readAvailable()); unsigned char *next = (unsigned char*)buffer; std::list<Segment>::const_iterator it = m_segments.begin(); while (pos != 0 && it != m_segments.end()) { if (pos < it->readAvailable()) break; pos -= it->readAvailable(); ++it; } MORDOR_ASSERT(it != m_segments.end()); for (; it != m_segments.end(); ++it) { size_t todo = (std::min)(length, it->readAvailable() - pos); memcpy(next, (char *)it->readBuffer().start() + pos, todo); next += todo; length -= todo; pos = 0; if (length == 0) break; } MORDOR_ASSERT(length == 0); }
void FiberCondition::broadcast() { boost::mutex::scoped_lock lock(m_mutex); if (m_waiters.empty()) return; boost::mutex::scoped_lock lock2(m_fiberMutex.m_mutex); std::list<std::pair<Scheduler *, Fiber::ptr> >::iterator it; for (it = m_waiters.begin(); it != m_waiters.end(); ++it) { std::pair<Scheduler *, Fiber::ptr> &next = *it; MORDOR_ASSERT(m_fiberMutex.m_owner != next.second); MORDOR_ASSERT(std::find(m_fiberMutex.m_waiters.begin(), m_fiberMutex.m_waiters.end(), next) == m_fiberMutex.m_waiters.end()); if (!m_fiberMutex.m_owner) { m_fiberMutex.m_owner = next.second; next.first->schedule(next.second); } else { m_fiberMutex.m_waiters.push_back(next); } } m_waiters.clear(); }
int Buffer::opCmp(const Buffer &rhs) const { std::list<Segment>::const_iterator leftIt, rightIt; int lengthResult = (int)((ptrdiff_t)readAvailable() - (ptrdiff_t)rhs.readAvailable()); leftIt = m_segments.begin(); rightIt = rhs.m_segments.begin(); size_t leftOffset = 0, rightOffset = 0; while (leftIt != m_segments.end() && rightIt != rhs.m_segments.end()) { MORDOR_ASSERT(leftOffset <= leftIt->readAvailable()); MORDOR_ASSERT(rightOffset <= rightIt->readAvailable()); size_t tocompare = (std::min)(leftIt->readAvailable() - leftOffset, rightIt->readAvailable() - rightOffset); if (tocompare == 0) break; int result = memcmp( (const unsigned char *)leftIt->readBuffer().start() + leftOffset, (const unsigned char *)rightIt->readBuffer().start() + rightOffset, tocompare); if (result != 0) return result; leftOffset += tocompare; rightOffset += tocompare; if (leftOffset == leftIt->readAvailable()) { leftOffset = 0; ++leftIt; } if (rightOffset == rightIt->readAvailable()) { rightOffset = 0; ++rightIt; } } return lengthResult; }
const std::vector<iovec> Buffer::readBuffers(size_t length) const { if (length == (size_t)~0) length = readAvailable(); MORDOR_ASSERT(length <= readAvailable()); std::vector<iovec> result; result.reserve(m_segments.size()); size_t remaining = length; std::list<Segment>::const_iterator it; for (it = m_segments.begin(); it != m_segments.end(); ++it) { size_t toConsume = (std::min)(it->readAvailable(), remaining); SegmentData data = it->readBuffer().slice(0, toConsume); #ifdef WINDOWS while (data.length() > 0) { iovec wsabuf; wsabuf.iov_base = (void *)data.start(); wsabuf.iov_len = iovLength(data.length()); result.push_back(wsabuf); data = data.slice(wsabuf.iov_len); } #else iovec iov; iov.iov_base = (void *)data.start(); iov.iov_len = data.length(); result.push_back(iov); #endif remaining -= toConsume; if (remaining == 0) break; } MORDOR_ASSERT(remaining == 0); invariant(); return result; }
void ZlibStream::flush(int flush) { flushBuffer(); while (true) { if (m_outBuffer.writeAvailable() == 0) m_outBuffer.reserve(m_bufferSize); struct iovec outbuf = m_outBuffer.writeBuffer(~0u, false); MORDOR_ASSERT(m_strm.avail_in == 0); m_strm.next_out = (Bytef*)outbuf.iov_base; m_strm.avail_out = outbuf.iov_len; int rc = deflate(&m_strm, flush); MORDOR_ASSERT(m_strm.avail_in == 0); MORDOR_LOG_DEBUG(g_log) << this << " deflate((0, " << outbuf.iov_len << "), " << flush << "): " << rc << " (0, " << m_strm.avail_out << ")"; m_outBuffer.produce(outbuf.iov_len - m_strm.avail_out); MORDOR_ASSERT(flush == Z_FINISH || rc != Z_STREAM_END); switch (rc) { case Z_STREAM_END: m_closed = true; deflateEnd(&m_strm); flushBuffer(); return; case Z_OK: break; case Z_BUF_ERROR: flushBuffer(); return; default: MORDOR_NOTREACHED(); } } }
FiberMutex::~FiberMutex() { #ifdef DEBUG boost::mutex::scoped_lock scopeLock(m_mutex); MORDOR_ASSERT(!m_owner); MORDOR_ASSERT(m_waiters.empty()); #endif }
void Buffer::copyIn(const Buffer &buffer, size_t length, size_t pos) { if (pos > buffer.readAvailable()) MORDOR_THROW_EXCEPTION(std::out_of_range("position out of range")); if (length == (size_t)~0) length = buffer.readAvailable() - pos; MORDOR_ASSERT(buffer.readAvailable() >= length + pos); invariant(); if (length == 0) return; // Split any mixed read/write bufs if (m_writeIt != m_segments.end() && m_writeIt->readAvailable() != 0) { m_segments.insert(m_writeIt, Segment(m_writeIt->readBuffer())); m_writeIt->consume(m_writeIt->readAvailable()); invariant(); } std::list<Segment>::const_iterator it = buffer.m_segments.begin(); while (pos != 0 && it != buffer.m_segments.end()) { if (pos < it->readAvailable()) break; pos -= it->readAvailable(); ++it; } MORDOR_ASSERT(it != buffer.m_segments.end()); for (; it != buffer.m_segments.end(); ++it) { size_t toConsume = (std::min)(it->readAvailable() - pos, length); if (m_readAvailable != 0 && it == buffer.m_segments.begin()) { std::list<Segment>::iterator previousIt = m_writeIt; --previousIt; if ((char *)previousIt->readBuffer().start() + previousIt->readBuffer().length() == (char *)it->readBuffer().start() + pos && previousIt->m_data.m_array.get() == it->m_data.m_array.get()) { MORDOR_ASSERT(previousIt->writeAvailable() == 0); previousIt->extend(toConsume); m_readAvailable += toConsume; length -= toConsume; pos = 0; if (length == 0) break; continue; } } Segment newSegment = Segment(it->readBuffer().slice(pos, toConsume)); m_segments.insert(m_writeIt, newSegment); m_readAvailable += toConsume; length -= toConsume; pos = 0; if (length == 0) break; } MORDOR_ASSERT(length == 0); MORDOR_ASSERT(readAvailable() >= length); }
void Buffer::Segment::truncate(size_t length) { MORDOR_ASSERT(length <= readAvailable()); MORDOR_ASSERT(m_writeIndex = readAvailable()); m_writeIndex = length; m_data = m_data.slice(0, length); invariant(); }
ptrdiff_t Buffer::find(const std::string &string, size_t length) const { if (length == (size_t)~0) length = readAvailable(); MORDOR_ASSERT(length <= readAvailable()); MORDOR_ASSERT(!string.empty()); size_t totalLength = 0; size_t foundSoFar = 0; std::list<Segment>::const_iterator it; for (it = m_segments.begin(); it != m_segments.end(); ++it) { const void *start = it->readBuffer().start(); size_t toscan = (std::min)(length, it->readAvailable()); while (toscan > 0) { if (foundSoFar == 0) { const void *point = memchr(start, string[0], toscan); if (point != NULL) { foundSoFar = 1; size_t found = (unsigned char*)point - (unsigned char*)start; toscan -= found + 1; length -= found + 1; totalLength += found; start = (unsigned char*)point + 1; } else { totalLength += toscan; length -= toscan; toscan = 0; continue; } } MORDOR_ASSERT(foundSoFar != 0); size_t tocompare = (std::min)(toscan, string.size() - foundSoFar); if (memcmp(start, string.c_str() + foundSoFar, tocompare) == 0) { foundSoFar += tocompare; toscan -= tocompare; length -= tocompare; if (foundSoFar == string.size()) break; } else { totalLength += foundSoFar; foundSoFar = 0; } } if (foundSoFar == string.size()) break; if (length == 0) break; } if (foundSoFar == string.size()) return totalLength; return -1; }
void Multipart::finish() { MORDOR_ASSERT(m_stream->supportsWrite()); MORDOR_ASSERT(!m_finished); std::string finalBoundary = m_boundary + "--\r\n"; m_stream->write(finalBoundary.c_str(), finalBoundary.size()); m_finished = true; if (multipartFinished) multipartFinished(); }
Connection::Connection(Stream::ptr stream) : m_stream(stream) { MORDOR_ASSERT(stream); MORDOR_ASSERT(stream->supportsRead()); MORDOR_ASSERT(stream->supportsWrite()); if (!stream->supportsUnread() || !stream->supportsFind()) { BufferedStream *buffered = new BufferedStream(stream); buffered->allowPartialReads(true); m_stream.reset(buffered); } }
void Scheduler::yieldTo(bool yieldToCallerOnTerminate) { MORDOR_ASSERT(t_fiber.get()); MORDOR_ASSERT(Scheduler::getThis() == this); if (yieldToCallerOnTerminate) MORDOR_ASSERT(m_rootThread == gettid()); if (t_fiber->state() != Fiber::HOLD) { m_stopping = m_autoStop || m_stopping; t_fiber->reset(); } t_fiber->yieldTo(yieldToCallerOnTerminate); }
const Buffer::SegmentData Buffer::SegmentData::slice(size_t start, size_t length) const { if (length == (size_t)~0) length = this->length() - start; MORDOR_ASSERT(start <= this->length()); MORDOR_ASSERT(length + start <= this->length()); SegmentData result; result.m_array = m_array; result.start((unsigned char*)this->start() + start); result.length(length); return result; }
Stream::ptr Connection::getStream(const GeneralHeaders &general, const EntityHeaders &entity, const std::string &method, Status status, boost::function<void()> notifyOnEof, boost::function<void()> notifyOnException, bool forRead) { MORDOR_ASSERT(hasMessageBody(general, entity, method, status)); Stream::ptr stream; if (forRead) { stream.reset(new SingleplexStream(m_stream, SingleplexStream::READ, false)); } else { stream.reset(new SingleplexStream(m_stream, SingleplexStream::WRITE, false)); } Stream::ptr baseStream(stream); for (ParameterizedList::const_reverse_iterator it(general.transferEncoding.rbegin()); it != general.transferEncoding.rend(); ++it) { if (stricmp(it->value.c_str(), "chunked") == 0) { stream.reset(new ChunkedStream(stream)); } else if (stricmp(it->value.c_str(), "deflate") == 0) { stream.reset(new ZlibStream(stream)); } else if (stricmp(it->value.c_str(), "gzip") == 0 || stricmp(it->value.c_str(), "x-gzip") == 0) { stream.reset(new GzipStream(stream)); } else if (stricmp(it->value.c_str(), "compress") == 0 || stricmp(it->value.c_str(), "x-compress") == 0) { MORDOR_ASSERT(false); } else if (stricmp(it->value.c_str(), "identity") == 0) { MORDOR_ASSERT(false); } else { MORDOR_ASSERT(false); } } if (stream != baseStream) { } else if (entity.contentLength != ~0ull) { LimitedStream::ptr limited(new LimitedStream(stream, entity.contentLength)); limited->strict(true); stream = limited; } else if (entity.contentType.type == "multipart") { // Getting stream to pass to multipart; self-delimiting } else { // Delimited by closing the connection } NotifyStream::ptr notify(new NotifyStream(stream)); stream = notify; notify->notifyOnClose = notifyOnEof; notify->notifyOnEof = notifyOnEof; notify->notifyOnException = notifyOnException; return stream; }
Multipart::Multipart(Stream::ptr stream, std::string boundary) : m_stream(stream), m_boundary(boundary), m_finished(false) { MORDOR_ASSERT(m_stream); MORDOR_ASSERT(m_stream->supportsRead() || m_stream->supportsWrite()); MORDOR_ASSERT(!(m_stream->supportsRead() && m_stream->supportsWrite())); while (!m_boundary.empty() && m_boundary[m_boundary.size() - 1] == ' ') m_boundary.resize(m_boundary.size() - 1); MORDOR_ASSERT(!m_boundary.empty()); MORDOR_ASSERT(m_boundary.size() <= 70); if (m_boundary.find_first_not_of(allowedBoundaryChars) != std::string::npos) { if (stream->supportsWrite()) { MORDOR_ASSERT(false); } else { MORDOR_THROW_EXCEPTION(InvalidMultipartBoundaryException()); } } m_boundary = "\r\n--" + m_boundary; if (m_stream->supportsRead()) { MORDOR_ASSERT(m_stream->supportsFind()); MORDOR_ASSERT(m_stream->supportsUnread()); } }
bool Scheduler::scheduleNoLock(boost::function<void ()> dg, tid_t thread) { MORDOR_LOG_DEBUG(g_log) << this << " scheduling " << dg << " on thread " << thread; MORDOR_ASSERT(dg); // Not thread-targeted, or this scheduler owns the targetted thread MORDOR_ASSERT(thread == emptytid() || thread == m_rootThread || contains(m_threads, thread)); FiberAndThread ft = {Fiber::ptr(), dg, thread }; bool tickleMe = m_fibers.empty(); m_fibers.push_back(ft); return tickleMe; }
long long BufferedStream::seek(long long offset, Anchor anchor) { MORDOR_ASSERT(parent()->supportsTell()); long long parentPos = parent()->tell(); long long bufferedPos = parentPos - m_readBuffer.readAvailable() + m_writeBuffer.readAvailable(); long long parentSize = parent()->supportsSize() ? -1ll : parent()->size(); // Check for no change in position if ((offset == 0 && anchor == CURRENT) || (offset == bufferedPos && anchor == BEGIN) || (parentSize != -1ll && offset + parentSize == bufferedPos && anchor == END)) return bufferedPos; MORDOR_ASSERT(supportsSeek()); flush(false); MORDOR_ASSERT(m_writeBuffer.readAvailable() == 0u); switch (anchor) { case BEGIN: MORDOR_ASSERT(offset >= 0); if (offset >= bufferedPos && offset <= parentPos) { m_readBuffer.consume((size_t)(offset - bufferedPos)); return offset; } m_readBuffer.clear(); break; case CURRENT: if (offset > 0 && offset <= (long long)m_readBuffer.readAvailable()) { m_readBuffer.consume((size_t)offset); return bufferedPos + offset; } offset -= m_readBuffer.readAvailable(); m_readBuffer.clear(); break; case END: if (parentSize == -1ll) throw std::invalid_argument("Can't seek from end without known size"); if (parentSize + offset >= bufferedPos && parentSize + offset <= parentPos) { m_readBuffer.consume((size_t)(parentSize + offset - bufferedPos)); return parentSize + offset; } m_readBuffer.clear(); break; default: MORDOR_NOTREACHED(); } return parent()->seek(offset, anchor); }
void Scheduler::yieldTo() { Scheduler *self = Scheduler::getThis(); MORDOR_ASSERT(self); MORDOR_LOG_DEBUG(g_log) << self << " yielding to scheduler"; MORDOR_ASSERT(t_fiber.get()); if (self->m_rootThread == gettid() && (t_fiber->state() == Fiber::INIT || t_fiber->state() == Fiber::TERM)) { self->m_callingFiber = Fiber::getThis(); self->yieldTo(true); } else { self->yieldTo(false); } }
FiberEvent::~FiberEvent() { #ifdef DEBUG boost::mutex::scoped_lock lock(m_mutex); MORDOR_ASSERT(m_waiters.empty()); #endif }
void Buffer::visit(boost::function<void (const void *, size_t)> dg, size_t length) const { if (length == (size_t)~0) length = readAvailable(); MORDOR_ASSERT(length <= readAvailable()); std::list<Segment>::const_iterator it; for (it = m_segments.begin(); it != m_segments.end() && length > 0; ++it) { size_t todo = (std::min)(length, it->readAvailable()); MORDOR_ASSERT(todo != 0); dg(it->readBuffer().start(), todo); length -= todo; } MORDOR_ASSERT(length == 0); }
Scheduler::~Scheduler() { MORDOR_ASSERT(m_stopping); if (getThis() == this) { t_scheduler = NULL; } }
FiberSemaphore::~FiberSemaphore() { #ifdef DEBUG boost::mutex::scoped_lock scopeLock(m_mutex); MORDOR_ASSERT(m_waiters.empty()); #endif }
void Scheduler::yield() { MORDOR_ASSERT(Scheduler::getThis()); Scheduler::getThis()->schedule(Fiber::getThis()); yieldTo(); }
bool Timer::reset(unsigned long long us, bool fromNow) { boost::mutex::scoped_lock lock(m_manager->m_mutex); if (!m_dg) return false; // No change if (us == m_us && !fromNow) return true; std::set<Timer::ptr, Timer::Comparator>::iterator it = m_manager->m_timers.find(shared_from_this()); MORDOR_ASSERT(it != m_manager->m_timers.end()); m_manager->m_timers.erase(it); unsigned long long start; if (fromNow) start = TimerManager::now(); else start = m_next - m_us; m_us = us; m_next = start + m_us; it = m_manager->m_timers.insert(shared_from_this()).first; bool atFront = (it == m_manager->m_timers.begin()) && !m_manager->m_tickled; if (atFront) m_manager->m_tickled = true; lock.unlock(); MORDOR_LOG_DEBUG(g_log) << this << " reset to " << m_us; if (atFront) m_manager->onTimerInsertedAtFront(); return true; }
static unsigned long long queryFrequency() { LARGE_INTEGER frequency; BOOL bRet = QueryPerformanceFrequency(&frequency); MORDOR_ASSERT(bRet); return (unsigned long long)frequency.QuadPart; }
TimerManager::~TimerManager() { #ifndef NDEBUG boost::mutex::scoped_lock lock(m_mutex); MORDOR_ASSERT(m_timers.empty()); #endif }