int psmx_wait_wait(struct fid_wait *wait, int timeout)
{
struct psmx_fid_wait *wait_priv;
int err = 0;
wait_priv = container_of(wait, struct psmx_fid_wait, wait.fid);
switch (wait_priv->type) {
case FI_WAIT_UNSPEC:
/* TODO: optimized custom wait */
break;
case FI_WAIT_FD:
err = fi_poll_fd(wait_priv->fd[0], timeout);
if (err > 0)
err = 0;
else if (err == 0)
err = -FI_ETIMEDOUT;
break;
case FI_WAIT_MUTEX_COND:
err = fi_wait_cond(&wait_priv->cond,
&wait_priv->mutex, timeout);
break;
default:
break;
}
return err;
}
static ssize_t
__fi_eq_cm_condread_data(struct fid_eq *eq, void *buf, size_t len, const void *cond)
{
ssize_t rc, cur, left;
ssize_t threshold;
struct __fid_eq_cm *_eq;
struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry *) buf;
_eq = container_of(eq, struct __fid_eq_cm, eq_fid);
threshold = (_eq->flags & FI_EQ_ATTR_COND) ? (long) cond : sizeof(*entry);
for(cur = 0, left = len; cur < threshold && left > 0; ) {
rc = __fi_eq_cm_read_data(eq, (void*)entry, left);
if (rc < 0)
return rc;
if (rc > 0) {
left -= rc;
entry += (rc / sizeof(*entry));
cur += rc;
}
if (cur >= threshold || left <= 0)
break;
fi_poll_fd(_eq->channel->fd);
}
return cur;
}
static int sock_ep_cm_send_msg(int sock_fd,
const struct sockaddr_in *addr, void *msg, size_t len)
{
int ret, retry = 0;
unsigned char response;
struct sockaddr_in from_addr;
socklen_t addr_len;
char sa_ip[INET_ADDRSTRLEN] = {0};
memcpy(sa_ip, inet_ntoa(addr->sin_addr), INET_ADDRSTRLEN);
SOCK_LOG_INFO("Sending message to %s:%d\n",
sa_ip, ntohs(addr->sin_port));
while (retry < SOCK_EP_MAX_RETRY) {
ret = sendto(sock_fd, (char *)msg, len, 0,
(struct sockaddr *) addr, sizeof *addr);
SOCK_LOG_INFO("Total Sent: %d\n", ret);
if (ret < 0)
return -1;
ret = fi_poll_fd(sock_fd, SOCK_CM_COMM_TIMEOUT);
retry++;
if (ret <= 0) {
continue;
}
addr_len = sizeof(struct sockaddr_in);
ret = recvfrom(sock_fd, &response, sizeof(response), 0,
(struct sockaddr *) &from_addr, &addr_len);
SOCK_LOG_INFO("Received ACK: %d\n", ret);
if (ret == sizeof(response))
return 0;
}
return -1;
}
static ssize_t __fi_eq_cm_read_data(struct fid_eq *eq, void *buf, size_t len)
{
struct __fid_eq_cm *_eq;
struct fi_eq_cm_entry *entry;
struct rdma_cm_event *event;
size_t left;
ssize_t ret = -FI_EINVAL;
_eq = container_of(eq, struct __fid_eq_cm, eq_fid);
entry = (struct fi_eq_cm_entry *) buf;
if (_eq->err.err)
return -FI_EAVAIL;
for (left = len; left >= sizeof(*entry); ) {
ret = rdma_get_cm_event(_eq->channel, &event);
if (!ret) {
ret = __fi_eq_cm_process_event(_eq, event, entry, left);
rdma_ack_cm_event(event);
if (ret < 0)
break;
else if (!ret)
continue;
left -= ret;
entry = ((void *) entry) + ret;
} else if (errno == EAGAIN) {
if (left < len)
return len - left;
if (!(_eq->flags & FI_BLOCK))
return 0;
fi_poll_fd(_eq->channel->fd);
} else {
ret = -errno;
break;
}
}
return (left < len) ? len - left : ret;
}
static void *sock_msg_ep_listener_thread (void *data)
{
struct sock_ep *ep = (struct sock_ep *)data;
struct sock_conn_response *conn_response = NULL;
struct fi_eq_cm_entry cm_entry;
struct fi_eq_err_entry cm_err_entry;
struct sockaddr_in from_addr;
socklen_t addr_len;
int ret, user_data_sz;
struct fid_ep *fid_ep;
struct sock_ep *sock_ep;
SOCK_LOG_INFO("Starting listener thread for EP: %p\n", ep);
ep->do_listen = 1;
while((volatile int)ep->do_listen) {
ret = fi_poll_fd(ep->socket, -1);
if (ret <= 0)
continue;
if (conn_response == NULL) {
conn_response = (struct sock_conn_response*)
calloc(1, sizeof(*conn_response) +
SOCK_EP_MAX_CM_DATA_SZ);
if (!conn_response) {
SOCK_LOG_ERROR("cannot allocate\n");
return NULL;
}
}
addr_len = sizeof(struct sockaddr_in);
ret = recvfrom(ep->socket, (char*)conn_response,
sizeof(*conn_response) + SOCK_EP_MAX_CM_DATA_SZ,
0, (struct sockaddr *) &from_addr, &addr_len);
if (ret <= 0)
continue;
SOCK_LOG_INFO("Total received: %d\n", ret);
if (ret < sizeof(*conn_response) ||
!sock_ep_cm_send_ack(ep->socket, &from_addr))
continue;
user_data_sz = 0;
switch (conn_response->hdr.type) {
case SOCK_CONN_ACCEPT:
SOCK_LOG_INFO("Received SOCK_CONN_ACCEPT\n");
memset(&cm_entry, 0, sizeof(cm_entry));
cm_entry.fid = conn_response->hdr.c_fid;
if (ret > sizeof(struct sock_conn_response)) {
user_data_sz = ret -
sizeof(struct sock_conn_response);
memcpy(&cm_entry.data,
(char *)conn_response +
sizeof(struct sock_conn_response),
user_data_sz);
}
fid_ep = container_of(conn_response->hdr.c_fid,
struct fid_ep, fid);
sock_ep = container_of(fid_ep, struct sock_ep, ep);
sock_ep->connected = 1;
sock_ep_enable(&ep->ep);
if (sock_eq_report_event(ep->eq, FI_CONNECTED, &cm_entry,
sizeof(cm_entry) + user_data_sz, 0))
SOCK_LOG_ERROR("Error in writing to EQ\n");
break;
case SOCK_CONN_REJECT:
SOCK_LOG_INFO("Received SOCK_CONN_REJECT\n");
memset(&cm_err_entry, 0, sizeof(cm_err_entry));
cm_err_entry.fid = conn_response->hdr.c_fid;
cm_err_entry.context = NULL;
cm_err_entry.data = 0;
cm_err_entry.err = -FI_ECONNREFUSED;
cm_err_entry.prov_errno = 0;
cm_err_entry.err_data = NULL;
if (ret > sizeof(struct sock_conn_response)) {
user_data_sz = ret -
sizeof(struct sock_conn_response);
memcpy(&cm_entry.data,
(char *)conn_response +
sizeof(struct sock_conn_response),
user_data_sz);
}
if (sock_eq_report_event(ep->eq, FI_ECONNREFUSED,
&cm_err_entry,
sizeof (cm_err_entry) +
user_data_sz, 0))
SOCK_LOG_ERROR("Error in writing to EQ\n");
goto out;
default:
SOCK_LOG_ERROR("Invalid event\n");
break;
}
conn_response = NULL;
}
out:
if (conn_response)
free(conn_response);
close(ep->socket);
ep->socket = 0;
return NULL;
}