static void *_sock_conn_listen(void *arg)
{
int conn_fd, ret;
char tmp;
socklen_t addr_size;
struct sockaddr_in remote;
struct pollfd poll_fds[2];
struct sock_ep_attr *ep_attr = (struct sock_ep_attr *)arg;
struct sock_conn_listener *listener = &ep_attr->listener;
struct sock_conn_map *map = &ep_attr->cmap;
poll_fds[0].fd = listener->sock;
poll_fds[1].fd = listener->signal_fds[1];
poll_fds[0].events = poll_fds[1].events = POLLIN;
listener->is_ready = 1;
while (listener->do_listen) {
if (poll(poll_fds, 2, -1) > 0) {
if (poll_fds[1].revents & POLLIN) {
ret = ofi_read_socket(listener->signal_fds[1], &tmp, 1);
if (ret != 1) {
SOCK_LOG_ERROR("Invalid signal\n");
goto err;
}
continue;
}
} else {
goto err;
}
addr_size = sizeof(remote);
conn_fd = accept(listener->sock, (struct sockaddr *) &remote,
&addr_size);
SOCK_LOG_DBG("CONN: accepted conn-req: %d\n", conn_fd);
if (conn_fd < 0) {
SOCK_LOG_ERROR("failed to accept: %s\n", strerror(errno));
goto err;
}
SOCK_LOG_DBG("ACCEPT: %s, %d\n", inet_ntoa(remote.sin_addr),
ntohs(remote.sin_port));
fastlock_acquire(&map->lock);
sock_conn_map_insert(ep_attr, &remote, conn_fd, 1);
fastlock_release(&map->lock);
sock_pe_signal(ep_attr->domain->pe);
}
err:
ofi_close_socket(listener->sock);
SOCK_LOG_DBG("Listener thread exited\n");
return NULL;
}
static void *util_ns_name_server_func(void *args)
{
struct util_ns *ns;
struct addrinfo hints = {
.ai_flags = AI_PASSIVE,
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
struct addrinfo *res, *p;
void *cleanup_args[2];
char *service;
SOCKET listenfd = INVALID_SOCKET, connfd;
int n, ret;
struct util_ns_cmd cmd = (const struct util_ns_cmd){ 0 };
ns = (struct util_ns *)args;
if (asprintf(&service, "%d", ns->ns_port) < 0)
return NULL;
n = getaddrinfo(NULL, service, &hints, &res);
if (n < 0) {
free(service);
return NULL;
}
for (p = res; p; p = p->ai_next) {
listenfd = ofi_socket(p->ai_family, p->ai_socktype,
p->ai_protocol);
if (listenfd != INVALID_SOCKET) {
n = 1;
(void) setsockopt(listenfd, SOL_SOCKET,
SO_REUSEADDR, &n, sizeof(n));
if (!bind(listenfd, p->ai_addr, p->ai_addrlen))
break;
ofi_close_socket(listenfd);
listenfd = INVALID_SOCKET;
}
}
freeaddrinfo(res);
free(service);
if (listenfd == INVALID_SOCKET)
return NULL;
if (util_ns_map_init(ns))
goto done;
ret = listen(listenfd, 256);
if (ret)
goto done;
cleanup_args[0] = (void *)(uintptr_t)listenfd;
cleanup_args[1] = (void *)ns;
pthread_cleanup_push(util_ns_name_server_cleanup,
(void *)cleanup_args);
while (1) {
connfd = accept(listenfd, NULL, 0);
if (connfd != INVALID_SOCKET) {
/* Read service data */
ret = ofi_read_socket(connfd, &cmd, cmd_len);
if (ret == cmd_len) {
(void) util_ns_op_dispatcher(ns, &cmd,
connfd);
}
ofi_close_socket(connfd);
}
}
pthread_cleanup_pop(1);
done:
ofi_close_socket(listenfd);
return NULL;
}
/*
* Name server API: client side
*/
static int util_ns_connect_server(struct util_ns *ns, const char *server)
{
struct addrinfo hints = {
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
struct addrinfo *res, *p;
char *service;
SOCKET sockfd = INVALID_SOCKET;
int n;
if (asprintf(&service, "%d", ns->ns_port) < 0)
return -1;
n = getaddrinfo(server, service, &hints, &res);
if (n < 0) {
free(service);
return -1;
}
for (p = res; p; p = p->ai_next) {
sockfd = ofi_socket(p->ai_family, p->ai_socktype, p->ai_protocol);
if (sockfd != INVALID_SOCKET) {
if (!connect(sockfd, p->ai_addr, p->ai_addrlen))
break;
ofi_close_socket(sockfd);
sockfd = INVALID_SOCKET;
}
}
freeaddrinfo(res);
free(service);
return sockfd;
}
int ofi_ns_add_local_name(struct util_ns *ns, void *service, void *name)
{
SOCKET sockfd;
int ret;
char *server = (ns->ns_hostname ?
ns->ns_hostname : OFI_NS_DEFAULT_HOSTNAME);
void *write_buf;
size_t write_len = 0;
struct util_ns_cmd cmd = {
.op = OFI_UTIL_NS_ADD,
.status = 0,
};
write_buf = calloc(cmd_len + ns->service_len + ns->name_len, 1);
if (!write_buf) {
ret = -FI_ENOMEM;
goto err1;
}
memcpy(write_buf, &cmd, cmd_len);
write_len += cmd_len;
memcpy((void *)((char *)write_buf + write_len), service,
ns->service_len);
write_len += ns->service_len;
memcpy((void *)((char *)write_buf + write_len), name,
ns->name_len);
write_len += ns->name_len;
sockfd = util_ns_connect_server(ns, server);
if (sockfd == INVALID_SOCKET) {
ret = -FI_ENODATA;
goto err2;
}
ret = util_ns_write_socket_op(sockfd, write_buf, write_len);
ret = ((ret == write_len) ? FI_SUCCESS : -FI_ENODATA);
ofi_close_socket(sockfd);
err2:
free(write_buf);
err1:
return ret;
}
int ofi_ns_del_local_name(struct util_ns *ns, void *service, void *name)
{
SOCKET sockfd;
int ret;
const char *server_hostname = (ns->ns_hostname ?
ns->ns_hostname : OFI_NS_DEFAULT_HOSTNAME);
void *write_buf;
size_t write_len = 0;
struct util_ns_cmd cmd = {
.op = OFI_UTIL_NS_DEL,
.status = 0,
};
write_buf = calloc(cmd_len + ns->service_len + ns->name_len, 1);
if (!write_buf) {
ret = -FI_ENOMEM;
goto err1;
}
memcpy(write_buf, &cmd, cmd_len);
write_len += cmd_len;
memcpy((void *)((char *)write_buf + write_len), service,
ns->service_len);
write_len += ns->service_len;
memcpy((void *)((char *)write_buf + write_len), name,
ns->name_len);
write_len += ns->name_len;
sockfd = util_ns_connect_server(ns, server_hostname);
if (sockfd == INVALID_SOCKET) {
ret = -FI_ENODATA;
goto err2;
}
ret = util_ns_write_socket_op(sockfd, write_buf, write_len);
ret = ((ret == write_len) ? FI_SUCCESS : -FI_ENODATA);
ofi_close_socket(sockfd);
err2:
free(write_buf);
err1:
return ret;
}
void *ofi_ns_resolve_name(struct util_ns *ns, const char *server_hostname,
void *service)
{
void *dest_addr = NULL, *io_buf;
size_t io_len = 0;
SOCKET sockfd;
ssize_t ret = 0;
struct util_ns_cmd cmd = {
.op = OFI_UTIL_NS_QUERY,
.status = 0,
};
sockfd = util_ns_connect_server(ns, server_hostname);
if (sockfd == INVALID_SOCKET)
goto err1;
io_buf = calloc(cmd_len + ns->service_len, 1);
if (!io_buf)
goto err2;
memcpy(io_buf, &cmd, cmd_len);
io_len += cmd_len;
memcpy((void *)((char *)io_buf + io_len), service,
ns->service_len);
io_len += ns->service_len;
ret = util_ns_write_socket_op(sockfd, io_buf, io_len);
if (ret < 0)
goto err3;
free(io_buf);
io_len = ns->service_len + ns->name_len;
io_buf = calloc(io_len, 1);
if (!io_buf)
goto err2;
ret = util_ns_read_socket_op(sockfd, &cmd, cmd_len);
if (ret < 0 || cmd.status)
goto err3;
ret = util_ns_read_socket_op(sockfd, io_buf, io_len);
if (ret == io_len) {
dest_addr = calloc(ns->name_len, 1);
if (!dest_addr)
goto err3;
io_len = 0;
memcpy(service, (void *)((char *)io_buf + io_len),
ns->service_len);
io_len += ns->service_len;
memcpy(dest_addr, (void *)((char *)io_buf + io_len),
ns->name_len);
}
err3:
free(io_buf);
err2:
ofi_close_socket(sockfd);
err1:
return dest_addr;
}
/*
* Name server API: server side
*/
void ofi_ns_start_server(struct util_ns *ns)
{
int ret;
SOCKET sockfd;
int sleep_usec = 1000;
char *server_hostname = (ns->ns_hostname ?
ns->ns_hostname : OFI_NS_DEFAULT_HOSTNAME);
ofi_osd_init();
ret = pthread_create(&ns->ns_thread, NULL,
util_ns_name_server_func, (void *)ns);
if (ret) {
/*
* use the main thread's ID as invalid
* value for the new thread
*/
ns->ns_thread = pthread_self();
}
/*
* Wait for the local name server to come up. It could be the thread
* created above, or the thread created by another process on the same
* node. The total wait time is about (1+2+4+...+8192)ms = 16 seconds.
*/
while (sleep_usec < 10000) {
sockfd = util_ns_connect_server(ns, server_hostname);
if (sockfd != INVALID_SOCKET) {
ofi_close_socket(sockfd);
return;
}
usleep(sleep_usec);
sleep_usec *= 2;
}
}
