void
RemoteNode::work_()
{
{
LOCK();
// the socket was created by another thread, per ZMQs documentation ownership
// transfer requires a full fence memory barrier which this lock should offer
LOG_INFO(node_id() << ": starting worker");
}
zmq::pollitem_t items[2];
bool wait_for_write = false;
while (true)
{
try
{
ASSERT(zock_ != nullptr);
items[0].socket = *zock_;
items[0].fd = -1;
items[0].events = ZMQ_POLLIN;
if (wait_for_write)
{
items[0].events |= ZMQ_POLLOUT;
}
items[1].socket = nullptr;
items[1].fd = event_fd_;
items[1].events = ZMQ_POLLIN;
const int ret = zmq::poll(items,
2,
-1);
THROW_WHEN(ret <= 0);
if (stop_)
{
LOCK();
ASSERT(queued_work_.empty());
ASSERT(submitted_work_.empty());
break;
}
else
{
if ((items[0].revents bitand ZMQ_POLLIN))
{
recv_responses_();
}
if ((items[0].revents bitand ZMQ_POLLOUT))
{
wait_for_write = not send_requests_();
}
if (items[1].revents bitand ZMQ_POLLIN)
{
wait_for_write = not send_requests_();
}
}
}
CATCH_STD_ALL_EWHAT({
LOG_ERROR(node_id() << ": caught exception in event loop: " << EWHAT << ". Resetting.");
init_zock_();
});
}
void
ClientNG::recv_(mdsproto::Tag txtag,
size_t txoff,
Read&& read)
{
mdsproto::ResponseHeader rxhdr;
client_.recv(ba::buffer(&rxhdr,
sizeof(rxhdr)),
timeout_);
if (rxhdr.magic != mdsproto::magic)
{
LOG_ERROR("Response lacks our protocol magic, giving up");
throw mdsproto::NoMagicException("no magic key in received header");
}
// LOG_TRACE("received hdr " << rxhdr.response_type <<
// ", tag " << rxhdr.tag <<
// ", size " << rxhdr.size <<
// ", flags " << rxhdr.flags);
TODO("AR: better exceptions");
THROW_WHEN(rxhdr.tag != txtag);
++in_counters_.messages;
in_counters_.data_bytes += rxhdr.size;
in_counters_.data_bytes_sqsum += rxhdr.size * rxhdr.size;
if (rxhdr.size)
{
if ((rxhdr.flags bitand mdsproto::ResponseHeader::Flags::UseShmem) == 0)
{
if (use_shmem_())
{
++in_counters_.shmem_overruns;
}
std::vector<capnp::word> rxbuf(rxhdr.size / sizeof(capnp::word));
client_.recv(ba::buffer(rxbuf),
timeout_);
capnp::FlatArrayMessageReader reader(kj::arrayPtr(rxbuf.data(),
rxbuf.size()));
handle_response_<T>(rxhdr,
reader,
std::move(read));
}
else
{
THROW_UNLESS(use_shmem_());
const uint8_t* addr = static_cast<const uint8_t*>(mr_->address()) + txoff;
THROW_UNLESS(addr + rxhdr.size <=
static_cast<const uint8_t*>(mr_->address()) + mr_->size());
auto seg(kj::arrayPtr(reinterpret_cast<const capnp::word*>(addr),
rxhdr.size / sizeof(capnp::word)));
capnp::SegmentArrayMessageReader reader(kj::arrayPtr(&seg, 1));
handle_response_<T>(rxhdr,
reader,
std::move(read));
}
}
}
