static void *_sock_conn_listen(void *arg)
{
int conn_fd, ret;
char tmp;
socklen_t addr_size;
struct sockaddr_in remote;
struct pollfd poll_fds[2];
struct sock_ep_attr *ep_attr = (struct sock_ep_attr *)arg;
struct sock_conn_listener *listener = &ep_attr->listener;
struct sock_conn_map *map = &ep_attr->cmap;
poll_fds[0].fd = listener->sock;
poll_fds[1].fd = listener->signal_fds[1];
poll_fds[0].events = poll_fds[1].events = POLLIN;
listener->is_ready = 1;
while (listener->do_listen) {
if (poll(poll_fds, 2, -1) > 0) {
if (poll_fds[1].revents & POLLIN) {
ret = ofi_read_socket(listener->signal_fds[1], &tmp, 1);
if (ret != 1) {
SOCK_LOG_ERROR("Invalid signal\n");
goto err;
}
continue;
}
} else {
goto err;
}
addr_size = sizeof(remote);
conn_fd = accept(listener->sock, (struct sockaddr *) &remote,
&addr_size);
SOCK_LOG_DBG("CONN: accepted conn-req: %d\n", conn_fd);
if (conn_fd < 0) {
SOCK_LOG_ERROR("failed to accept: %s\n", strerror(errno));
goto err;
}
SOCK_LOG_DBG("ACCEPT: %s, %d\n", inet_ntoa(remote.sin_addr),
ntohs(remote.sin_port));
fastlock_acquire(&map->lock);
sock_conn_map_insert(ep_attr, &remote, conn_fd, 1);
fastlock_release(&map->lock);
sock_pe_signal(ep_attr->domain->pe);
}
err:
ofi_close_socket(listener->sock);
SOCK_LOG_DBG("Listener thread exited\n");
return NULL;
}
struct sock_conn *sock_ep_connect(struct sock_ep_attr *ep_attr, fi_addr_t index)
{
int conn_fd = -1, ret;
int do_retry = sock_conn_retry;
struct sock_conn *conn, *new_conn;
struct sockaddr_in addr;
socklen_t lon;
int valopt = 0;
struct pollfd poll_fd;
if (ep_attr->ep_type == FI_EP_MSG) {
/* Need to check that destination address has been
passed to endpoint */
assert(ep_attr->dest_addr);
addr = *ep_attr->dest_addr;
addr.sin_port = htons(ep_attr->msg_dest_port);
} else {
addr = *((struct sockaddr_in *)&ep_attr->av->table[index].addr);
}
do_connect:
fastlock_acquire(&ep_attr->cmap.lock);
conn = sock_ep_lookup_conn(ep_attr, index, &addr);
fastlock_release(&ep_attr->cmap.lock);
if (conn != SOCK_CM_CONN_IN_PROGRESS)
return conn;
conn_fd = ofi_socket(AF_INET, SOCK_STREAM, 0);
if (conn_fd == -1) {
SOCK_LOG_ERROR("failed to create conn_fd, errno: %d\n", errno);
errno = FI_EOTHER;
return NULL;
}
ret = fd_set_nonblock(conn_fd);
if (ret) {
SOCK_LOG_ERROR("failed to set conn_fd nonblocking, errno: %d\n", errno);
errno = FI_EOTHER;
ofi_close_socket(conn_fd);
return NULL;
}
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep_attr->src_addr->sin_addr));
ret = connect(conn_fd, (struct sockaddr *) &addr, sizeof addr);
if (ret < 0) {
if (ofi_sockerr() == EINPROGRESS) {
poll_fd.fd = conn_fd;
poll_fd.events = POLLOUT;
ret = poll(&poll_fd, 1, 15 * 1000);
if (ret < 0) {
SOCK_LOG_DBG("poll failed\n");
goto retry;
}
lon = sizeof(int);
ret = getsockopt(conn_fd, SOL_SOCKET, SO_ERROR, (void*)(&valopt), &lon);
if (ret < 0) {
SOCK_LOG_DBG("getsockopt failed: %d, %d\n", ret, conn_fd);
goto retry;
}
if (valopt) {
SOCK_LOG_DBG("Error in connection() %d - %s - %d\n", valopt, strerror(valopt), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep_attr->src_addr->sin_addr));
goto retry;
}
goto out;
} else {
SOCK_LOG_DBG("Timeout or error() - %s: %d\n", strerror(errno), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep_attr->src_addr->sin_addr));
goto retry;
}
} else {
goto out;
}
retry:
do_retry--;
sleep(10);
if (!do_retry)
goto err;
if (conn_fd != -1) {
ofi_close_socket(conn_fd);
conn_fd = -1;
}
SOCK_LOG_ERROR("Connect error, retrying - %s - %d\n", strerror(errno), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep_attr->src_addr->sin_addr));
goto do_connect;
out:
fastlock_acquire(&ep_attr->cmap.lock);
new_conn = sock_conn_map_insert(ep_attr, &addr, conn_fd, 0);
if (!new_conn) {
fastlock_release(&ep_attr->cmap.lock);
goto err;
}
new_conn->av_index = (ep_attr->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : index;
conn = ofi_idm_lookup(&ep_attr->av_idm, index);
if (conn == SOCK_CM_CONN_IN_PROGRESS) {
if (ofi_idm_set(&ep_attr->av_idm, index, new_conn) < 0)
SOCK_LOG_ERROR("ofi_idm_set failed\n");
conn = new_conn;
}
fastlock_release(&ep_attr->cmap.lock);
return conn;
err:
ofi_close_socket(conn_fd);
return NULL;
}
struct sock_conn *sock_ep_connect(struct sock_ep *ep, fi_addr_t index)
{
int conn_fd = -1, ret;
int do_retry = sock_conn_retry;
struct sock_conn *conn, *new_conn;
uint16_t idx;
struct sockaddr_in *addr;
socklen_t lon;
int valopt = 0;
struct pollfd poll_fd;
if (ep->ep_type == FI_EP_MSG) {
idx = 0;
addr = ep->dest_addr;
} else {
idx = index & ep->av->mask;
addr = (struct sockaddr_in *)&ep->av->table[idx].addr;
}
do_connect:
fastlock_acquire(&ep->cmap.lock);
conn = sock_ep_lookup_conn(ep, index, addr);
fastlock_release(&ep->cmap.lock);
if (conn != SOCK_CM_CONN_IN_PROGRESS)
return conn;
conn_fd = socket(AF_INET, SOCK_STREAM, 0);
if (conn_fd == -1) {
SOCK_LOG_ERROR("failed to create conn_fd, errno: %d\n", errno);
errno = FI_EOTHER;
return NULL;
}
ret = fd_set_nonblock(conn_fd);
if (ret) {
SOCK_LOG_ERROR("failed to set conn_fd nonblocking, errno: %d\n", errno);
errno = FI_EOTHER;
close(conn_fd);
return NULL;
}
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
ntohs(addr->sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep->src_addr->sin_addr));
ret = connect(conn_fd, (struct sockaddr *) addr, sizeof *addr);
if (ret < 0) {
if (errno == EINPROGRESS) {
poll_fd.fd = conn_fd;
poll_fd.events = POLLOUT;
ret = poll(&poll_fd, 1, 15 * 1000);
if (ret < 0) {
SOCK_LOG_DBG("poll failed\n");
goto retry;
}
lon = sizeof(int);
ret = getsockopt(conn_fd, SOL_SOCKET, SO_ERROR, (void*)(&valopt), &lon);
if (ret < 0) {
SOCK_LOG_DBG("getsockopt failed: %d, %d\n", ret, conn_fd);
goto retry;
}
if (valopt) {
SOCK_LOG_DBG("Error in connection() %d - %s - %d\n", valopt, strerror(valopt), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
ntohs(addr->sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep->src_addr->sin_addr));
goto retry;
}
goto out;
} else {
SOCK_LOG_DBG("Timeout or error() - %s: %d\n", strerror(errno), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
ntohs(addr->sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep->src_addr->sin_addr));
goto retry;
}
} else {
goto out;
}
retry:
do_retry--;
sleep(10);
if (!do_retry)
goto err;
if (conn_fd != -1) {
close(conn_fd);
conn_fd = -1;
}
SOCK_LOG_ERROR("Connect error, retrying - %s - %d\n", strerror(errno), conn_fd);
SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
ntohs(addr->sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
inet_ntoa(ep->src_addr->sin_addr));
goto do_connect;
out:
fastlock_acquire(&ep->cmap.lock);
new_conn = sock_conn_map_insert(ep, addr, conn_fd, 0);
new_conn->av_index = (ep->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : (fi_addr_t) idx;
conn = idm_lookup(&ep->av_idm, index);
if (conn == SOCK_CM_CONN_IN_PROGRESS) {
idm_set(&ep->av_idm, index, new_conn);
conn = new_conn;
}
fastlock_release(&ep->cmap.lock);
return conn;
err:
close(conn_fd);
return NULL;
}
