void asio_rpc_session::connect()
{
if (try_connecting())
{
boost::asio::ip::tcp::endpoint ep(
boost::asio::ip::address_v4(_remote_addr.ip()), _remote_addr.port());
add_ref();
_socket->async_connect(ep, [this](boost::system::error_code ec)
{
if (!ec)
{
dinfo("client session %s connected",
_remote_addr.to_string()
);
set_options();
set_connected();
on_send_completed();
do_read();
}
else
{
derror("client session connect to %s failed, error = %s",
_remote_addr.to_string(),
ec.message().c_str()
);
on_failure();
}
release_ref();
});
}
}
void asio_rpc_session::write(uint64_t signature)
{
std::vector<boost::asio::const_buffer> buffers2;
int bcount = (int)_sending_buffers.size();
// prepare buffers
buffers2.resize(bcount);
for (int i = 0; i < bcount; i++)
{
buffers2[i] = boost::asio::const_buffer(_sending_buffers[i].buf, _sending_buffers[i].sz);
}
add_ref();
boost::asio::async_write(*_socket, buffers2,
[this, signature](boost::system::error_code ec, std::size_t length)
{
if (!!ec)
{
derror("asio write failed: %s", ec.message().c_str());
on_failure();
}
else
{
on_send_completed(signature);
}
release_ref();
});
}
void hpc_rpc_session::connect()
{
if (!try_connecting())
return;
_connect_event.callback = [this](int err, uint32_t io_size, uintptr_t lpolp)
{
//dinfo("ConnectEx completed, err = %d, size = %u", err, io_size);
if (err != ERROR_SUCCESS)
{
dwarn("ConnectEx failed, err = %d", err);
this->on_failure();
}
else
{
dinfo("client session %s:%hu connected",
_remote_addr.name(),
_remote_addr.port()
);
set_connected();
on_send_completed(nullptr);
do_read();
}
this->release_ref(); // added before ConnectEx
};
memset(&_connect_event.olp, 0, sizeof(_connect_event.olp));
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(_remote_addr.ip());
addr.sin_port = htons(_remote_addr.port());
this->add_ref(); // released in _connect_event.callback
BOOL rt = s_lpfnConnectEx(
_socket,
(struct sockaddr*)&addr,
(int)sizeof(addr),
0,
0,
0,
&_connect_event.olp
);
if (!rt && (WSAGetLastError() != ERROR_IO_PENDING))
{
dwarn("ConnectEx failed, err = %d", ::WSAGetLastError());
this->release_ref();
on_failure();
}
}
void sim_server_session::send(uint64_t sig)
{
for (auto& msg : _sending_msgs)
{
message_ex* recv_msg = virtual_send_message(msg);
{
node_scoper ns(_client->net().node());
_client->on_recv_reply(recv_msg->header->id, recv_msg,
recv_msg->to_address == recv_msg->from_address ?
0 : (static_cast<sim_network_provider*>(&_net))->net_delay_milliseconds()
);
}
}
on_send_completed(sig);
}
void sim_client_session::send(uint64_t sig)
{
for (auto& msg : _sending_msgs)
{
sim_network_provider* rnet = nullptr;
if (!s_switch[task_spec::get(msg->local_rpc_code)->rpc_call_channel][msg->hdr_format].get(remote_address(), rnet))
{
derror("cannot find destination node %s in simulator",
remote_address().to_string()
);
//on_disconnected(); // disable this to avoid endless resending
}
else
{
auto server_session = rnet->get_server_session(_net.address());
if (nullptr == server_session)
{
rpc_session_ptr cptr = this;
message_parser_ptr parser(_net.new_message_parser(msg->hdr_format));
server_session = new sim_server_session(*rnet, _net.address(),
cptr, parser);
rnet->on_server_session_accepted(server_session);
}
message_ex* recv_msg = virtual_send_message(msg);
{
node_scoper ns(rnet->node());
bool ret = server_session->on_recv_message(recv_msg,
recv_msg->to_address == recv_msg->header->from_address ?
0 : rnet->net_delay_milliseconds()
);
dassert(ret, "");
}
}
}
on_send_completed(sig);
}
void hpc_rpc_session::do_safe_write(message_ex* msg)
{
utils::auto_lock<utils::ex_lock_nr> l(_send_lock);
if (nullptr == msg)
{
if (_sending_msg)
{
do_write(_sending_msg);
}
else
{
_send_lock.unlock(); // avoid recursion
on_send_completed(nullptr); // send next msg if there is.
_send_lock.lock();
}
}
else
{
do_write(msg);
}
}
void hpc_rpc_session::do_write(message_ex* msg)
{
add_ref();
_write_event.callback = [this](int err, uint32_t length, uintptr_t lolp)
{
dassert((LPOVERLAPPED)lolp == &_write_event.olp, "must be exact this overlapped");
if (err != ERROR_SUCCESS)
{
dwarn("WSASend failed, err = %d", err);
on_failure();
}
else
{
int len = (int)length;
int buf_i = _sending_buffer_start_index;
while (len > 0)
{
auto& buf = _sending_buffers[buf_i];
if (len >= (int)buf.sz)
{
buf_i++;
len -= (int)buf.sz;
}
else
{
buf.buf = (char*)buf.buf + len;
buf.sz -= (uint32_t)len;
break;
}
}
_sending_buffer_start_index = buf_i;
// message completed, continue next message
if (_sending_buffer_start_index == (int)_sending_buffers.size())
{
dassert(len == 0, "buffer must be sent completely");
auto lmsg = _sending_msg;
_sending_msg = nullptr;
on_send_completed(lmsg);
}
else
do_write(_sending_msg);
}
release_ref();
};
memset(&_write_event.olp, 0, sizeof(_write_event.olp));
// new msg
if (_sending_msg != msg)
{
dassert(_sending_msg == nullptr, "only one sending msg is possible");
_sending_msg = msg;
_sending_buffer_start_index = 0;
}
// continue old msg
else
{
// nothing to do
}
int buffer_count = (int)_sending_buffers.size() - _sending_buffer_start_index;
static_assert (sizeof(dsn_message_parser::send_buf) == sizeof(WSABUF), "make sure they are compatible");
DWORD bytes = 0;
int rt = WSASend(
_socket,
(LPWSABUF)&_sending_buffers[_sending_buffer_start_index],
(DWORD)buffer_count,
&bytes,
0,
&_write_event.olp,
NULL
);
if (SOCKET_ERROR == rt && (WSAGetLastError() != ERROR_IO_PENDING))
{
dwarn("WSASend failed, err = %d", ::WSAGetLastError());
release_ref();
on_failure();
}
//dinfo("WSASend called, err = %d", rt);
}
void hpc_rpc_session::do_write(message_ex* msg)
{
static_assert (sizeof(dsn_message_parser::send_buf) == sizeof(struct iovec),
"make sure they are compatible");
dbg_dassert(msg != nullptr, "cannot send empty msg");
// new msg
if (_sending_msg == nullptr)
{
_sending_msg = msg;
_sending_buffer_start_index = 0;
}
// continue old msg
else
{
dassert(_sending_msg == msg, "only one sending msg is possible");
}
// prepare send buffer, make sure header is already in the buffer
while (true)
{
int buffer_count = (int)_sending_buffers.size() - _sending_buffer_start_index;
struct msghdr hdr;
memset((void*)&hdr, 0, sizeof(hdr));
hdr.msg_name = (void*)&_peer_addr;
hdr.msg_namelen = (socklen_t)sizeof(_peer_addr);
hdr.msg_iov = (struct iovec*)&_sending_buffers[_sending_buffer_start_index];
hdr.msg_iovlen = (size_t)buffer_count;
int sz = sendmsg(_socket, &hdr, MSG_NOSIGNAL);
int err = errno;
dinfo("(s = %d) call sendmsg on %s:%hu, return %d, err = %s",
_socket,
_remote_addr.name(),
_remote_addr.port(),
sz,
strerror(err)
);
if (sz < 0)
{
if (err != EAGAIN && err != EWOULDBLOCK)
{
derror("(s = %d) sendmsg failed, err = %s", _socket, strerror(err));
on_failure();
}
else
{
// wait for epoll_wait notification
}
return;
}
else
{
int len = (int)sz;
int buf_i = _sending_buffer_start_index;
while (len > 0)
{
auto& buf = _sending_buffers[buf_i];
if (len >= (int)buf.sz)
{
buf_i++;
len -= (int)buf.sz;
}
else
{
buf.buf = (char*)buf.buf + len;
buf.sz -= len;
break;
}
}
_sending_buffer_start_index = buf_i;
// message completed, continue next message
if (_sending_buffer_start_index == (int)_sending_buffers.size())
{
dassert(len == 0, "buffer must be sent completely");
_sending_msg = nullptr;
_send_lock.unlock(); // avoid recursion
// try next msg recursively
on_send_completed(msg);
_send_lock.lock();
return;
}
else
{
// try next while(true) loop to continue sending current msg
}
}
}
}
