static void *_sock_conn_listen(void *arg) { int conn_fd, ret; char tmp; socklen_t addr_size; struct sockaddr_in remote; struct pollfd poll_fds[2]; struct sock_ep_attr *ep_attr = (struct sock_ep_attr *)arg; struct sock_conn_listener *listener = &ep_attr->listener; struct sock_conn_map *map = &ep_attr->cmap; poll_fds[0].fd = listener->sock; poll_fds[1].fd = listener->signal_fds[1]; poll_fds[0].events = poll_fds[1].events = POLLIN; listener->is_ready = 1; while (listener->do_listen) { if (poll(poll_fds, 2, -1) > 0) { if (poll_fds[1].revents & POLLIN) { ret = ofi_read_socket(listener->signal_fds[1], &tmp, 1); if (ret != 1) { SOCK_LOG_ERROR("Invalid signal\n"); goto err; } continue; } } else { goto err; } addr_size = sizeof(remote); conn_fd = accept(listener->sock, (struct sockaddr *) &remote, &addr_size); SOCK_LOG_DBG("CONN: accepted conn-req: %d\n", conn_fd); if (conn_fd < 0) { SOCK_LOG_ERROR("failed to accept: %s\n", strerror(errno)); goto err; } SOCK_LOG_DBG("ACCEPT: %s, %d\n", inet_ntoa(remote.sin_addr), ntohs(remote.sin_port)); fastlock_acquire(&map->lock); sock_conn_map_insert(ep_attr, &remote, conn_fd, 1); fastlock_release(&map->lock); sock_pe_signal(ep_attr->domain->pe); } err: ofi_close_socket(listener->sock); SOCK_LOG_DBG("Listener thread exited\n"); return NULL; }
struct sock_conn *sock_ep_connect(struct sock_ep_attr *ep_attr, fi_addr_t index) { int conn_fd = -1, ret; int do_retry = sock_conn_retry; struct sock_conn *conn, *new_conn; struct sockaddr_in addr; socklen_t lon; int valopt = 0; struct pollfd poll_fd; if (ep_attr->ep_type == FI_EP_MSG) { /* Need to check that destination address has been passed to endpoint */ assert(ep_attr->dest_addr); addr = *ep_attr->dest_addr; addr.sin_port = htons(ep_attr->msg_dest_port); } else { addr = *((struct sockaddr_in *)&ep_attr->av->table[index].addr); } do_connect: fastlock_acquire(&ep_attr->cmap.lock); conn = sock_ep_lookup_conn(ep_attr, index, &addr); fastlock_release(&ep_attr->cmap.lock); if (conn != SOCK_CM_CONN_IN_PROGRESS) return conn; conn_fd = ofi_socket(AF_INET, SOCK_STREAM, 0); if (conn_fd == -1) { SOCK_LOG_ERROR("failed to create conn_fd, errno: %d\n", errno); errno = FI_EOTHER; return NULL; } ret = fd_set_nonblock(conn_fd); if (ret) { SOCK_LOG_ERROR("failed to set conn_fd nonblocking, errno: %d\n", errno); errno = FI_EOTHER; ofi_close_socket(conn_fd); return NULL; } SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep_attr->src_addr->sin_addr)); ret = connect(conn_fd, (struct sockaddr *) &addr, sizeof addr); if (ret < 0) { if (ofi_sockerr() == EINPROGRESS) { poll_fd.fd = conn_fd; poll_fd.events = POLLOUT; ret = poll(&poll_fd, 1, 15 * 1000); if (ret < 0) { SOCK_LOG_DBG("poll failed\n"); goto retry; } lon = sizeof(int); ret = getsockopt(conn_fd, SOL_SOCKET, SO_ERROR, (void*)(&valopt), &lon); if (ret < 0) { SOCK_LOG_DBG("getsockopt failed: %d, %d\n", ret, conn_fd); goto retry; } if (valopt) { SOCK_LOG_DBG("Error in connection() %d - %s - %d\n", valopt, strerror(valopt), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep_attr->src_addr->sin_addr)); goto retry; } goto out; } else { SOCK_LOG_DBG("Timeout or error() - %s: %d\n", strerror(errno), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep_attr->src_addr->sin_addr)); goto retry; } } else { goto out; } retry: do_retry--; sleep(10); if (!do_retry) goto err; if (conn_fd != -1) { ofi_close_socket(conn_fd); conn_fd = -1; } SOCK_LOG_ERROR("Connect error, retrying - %s - %d\n", strerror(errno), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep_attr->src_addr->sin_addr)); goto do_connect; out: fastlock_acquire(&ep_attr->cmap.lock); new_conn = sock_conn_map_insert(ep_attr, &addr, conn_fd, 0); if (!new_conn) { fastlock_release(&ep_attr->cmap.lock); goto err; } new_conn->av_index = (ep_attr->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : index; conn = ofi_idm_lookup(&ep_attr->av_idm, index); if (conn == SOCK_CM_CONN_IN_PROGRESS) { if (ofi_idm_set(&ep_attr->av_idm, index, new_conn) < 0) SOCK_LOG_ERROR("ofi_idm_set failed\n"); conn = new_conn; } fastlock_release(&ep_attr->cmap.lock); return conn; err: ofi_close_socket(conn_fd); return NULL; }
struct sock_conn *sock_ep_connect(struct sock_ep *ep, fi_addr_t index) { int conn_fd = -1, ret; int do_retry = sock_conn_retry; struct sock_conn *conn, *new_conn; uint16_t idx; struct sockaddr_in *addr; socklen_t lon; int valopt = 0; struct pollfd poll_fd; if (ep->ep_type == FI_EP_MSG) { idx = 0; addr = ep->dest_addr; } else { idx = index & ep->av->mask; addr = (struct sockaddr_in *)&ep->av->table[idx].addr; } do_connect: fastlock_acquire(&ep->cmap.lock); conn = sock_ep_lookup_conn(ep, index, addr); fastlock_release(&ep->cmap.lock); if (conn != SOCK_CM_CONN_IN_PROGRESS) return conn; conn_fd = socket(AF_INET, SOCK_STREAM, 0); if (conn_fd == -1) { SOCK_LOG_ERROR("failed to create conn_fd, errno: %d\n", errno); errno = FI_EOTHER; return NULL; } ret = fd_set_nonblock(conn_fd); if (ret) { SOCK_LOG_ERROR("failed to set conn_fd nonblocking, errno: %d\n", errno); errno = FI_EOTHER; close(conn_fd); return NULL; } SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep->src_addr->sin_addr)); ret = connect(conn_fd, (struct sockaddr *) addr, sizeof *addr); if (ret < 0) { if (errno == EINPROGRESS) { poll_fd.fd = conn_fd; poll_fd.events = POLLOUT; ret = poll(&poll_fd, 1, 15 * 1000); if (ret < 0) { SOCK_LOG_DBG("poll failed\n"); goto retry; } lon = sizeof(int); ret = getsockopt(conn_fd, SOL_SOCKET, SO_ERROR, (void*)(&valopt), &lon); if (ret < 0) { SOCK_LOG_DBG("getsockopt failed: %d, %d\n", ret, conn_fd); goto retry; } if (valopt) { SOCK_LOG_DBG("Error in connection() %d - %s - %d\n", valopt, strerror(valopt), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep->src_addr->sin_addr)); goto retry; } goto out; } else { SOCK_LOG_DBG("Timeout or error() - %s: %d\n", strerror(errno), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep->src_addr->sin_addr)); goto retry; } } else { goto out; } retry: do_retry--; sleep(10); if (!do_retry) goto err; if (conn_fd != -1) { close(conn_fd); conn_fd = -1; } SOCK_LOG_ERROR("Connect error, retrying - %s - %d\n", strerror(errno), conn_fd); SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); SOCK_LOG_DBG("Connecting using address:%s\n", inet_ntoa(ep->src_addr->sin_addr)); goto do_connect; out: fastlock_acquire(&ep->cmap.lock); new_conn = sock_conn_map_insert(ep, addr, conn_fd, 0); new_conn->av_index = (ep->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : (fi_addr_t) idx; conn = idm_lookup(&ep->av_idm, index); if (conn == SOCK_CM_CONN_IN_PROGRESS) { idm_set(&ep->av_idm, index, new_conn); conn = new_conn; } fastlock_release(&ep->cmap.lock); return conn; err: close(conn_fd); return NULL; }