void rs_free_slave(void *data)
{
int err;
rs_slave_info_t *si;
si = data == NULL ? rs_slave_info : (rs_slave_info_t *) data;
if(si != NULL) {
if(si->io_thread != 0) {
if((err = pthread_cancel(si->io_thread)) != 0) {
rs_log_err(err, "pthread_cancel() failed, io_thread");
}
}
if(si->svr_fd != -1) {
rs_close(si->svr_fd);
}
if(si->redis_thread != 0) {
if((err = pthread_cancel(si->redis_thread)) != 0) {
rs_log_err(err, "pthread_cancel() failed, redis_thread");
}
}
if(si->c != NULL) {
redisFree(si->c);
}
if(si->ring_buf != NULL) {
rs_free_ring_buffer2(si->ring_buf);
free(si->ring_buf);
}
/* close slave info file */
if(si->info_fd != -1) {
rs_close(si->info_fd);
}
if((err = pthread_attr_destroy(&(si->thread_attr))) != 0) {
rs_log_err(err, "pthread_attr_destroy() failed, thread_attr");
}
rs_free_conf(&(si->conf));
free(si->conf.kv);
free(si);
}
rs_slave_info = NULL;
}
static int server_listen(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
int val, ret;
if (use_rgai) {
rai_hints.ai_flags |= RAI_PASSIVE;
ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai);
} else {
ai_hints.ai_flags |= AI_PASSIVE;
ret = getaddrinfo(src_addr, port, &ai_hints, &ai);
}
if (ret) {
perror("getaddrinfo");
return ret;
}
lrs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (lrs < 0) {
perror("rsocket");
ret = lrs;
goto free;
}
val = 1;
ret = rs_setsockopt(lrs, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val);
if (ret) {
perror("rsetsockopt SO_REUSEADDR");
goto close;
}
ret = rai ? rs_bind(lrs, rai->ai_src_addr, rai->ai_src_len) :
rs_bind(lrs, ai->ai_addr, ai->ai_addrlen);
if (ret) {
perror("rbind");
goto close;
}
ret = rs_listen(lrs, 1);
if (ret)
perror("rlisten");
close:
if (ret)
rs_close(lrs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
static void rs_free_accept_thread(void *data)
{
rs_master_info_t *mi;
mi = (rs_master_info_t *) data;
/* NOTICE : if reload signal, must skip send SIGQUIT */
if(mi != NULL) {
rs_log_info("free accept thread");
// kill(rs_pid, SIGQUIT);
rs_close(mi->svr_fd);
mi->svr_fd = -1;
}
}
static int run(void)
{
int i, ret = 0;
buf = malloc(!custom ? test_size[TEST_CNT - 1].size : transfer_size);
if (!buf) {
perror("malloc");
return -1;
}
if (!dst_addr) {
ret = server_listen();
if (ret)
goto free;
}
printf("%-10s%-8s%-8s%-8s%-8s%8s %10s%13s\n",
"name", "bytes", "xfers", "iters", "total", "time", "Gb/sec", "usec/xfer");
if (!custom) {
optimization = opt_latency;
ret = dst_addr ? client_connect() : server_connect();
if (ret)
goto free;
for (i = 0; i < TEST_CNT && !fork_pid; i++) {
if (test_size[i].option > size_option)
continue;
init_latency_test(test_size[i].size);
run_test();
}
if (fork_pid)
wait(NULL);
else
rs_shutdown(rs, SHUT_RDWR);
rs_close(rs);
if (!dst_addr && use_fork && !fork_pid)
goto free;
optimization = opt_bandwidth;
ret = dst_addr ? client_connect() : server_connect();
if (ret)
goto free;
for (i = 0; i < TEST_CNT && !fork_pid; i++) {
if (test_size[i].option > size_option)
continue;
init_bandwidth_test(test_size[i].size);
run_test();
}
} else {
ret = dst_addr ? client_connect() : server_connect();
if (ret)
goto free;
if (!fork_pid)
ret = run_test();
}
if (fork_pid)
wait(NULL);
else
rs_shutdown(rs, SHUT_RDWR);
rs_close(rs);
free:
free(buf);
return ret;
}
static int client_connect(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
struct pollfd fds;
int ret, err;
socklen_t len;
ret = use_rgai ? rdma_getaddrinfo(dst_addr, port, &rai_hints, &rai) :
getaddrinfo(dst_addr, port, &ai_hints, &ai);
if (ret) {
perror("getaddrinfo");
return ret;
}
rs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (rs < 0) {
perror("rsocket");
ret = rs;
goto free;
}
set_options(rs);
/* TODO: bind client to src_addr */
if (rai && rai->ai_route) {
ret = rs_setsockopt(rs, SOL_RDMA, RDMA_ROUTE, rai->ai_route,
rai->ai_route_len);
if (ret) {
perror("rsetsockopt RDMA_ROUTE");
goto close;
}
}
ret = rai ? rs_connect(rs, rai->ai_dst_addr, rai->ai_dst_len) :
rs_connect(rs, ai->ai_addr, ai->ai_addrlen);
if (ret && (errno != EINPROGRESS)) {
perror("rconnect");
goto close;
}
if (ret && (errno == EINPROGRESS)) {
fds.fd = rs;
fds.events = POLLOUT;
ret = do_poll(&fds, poll_timeout);
if (ret)
goto close;
len = sizeof err;
ret = rs_getsockopt(rs, SOL_SOCKET, SO_ERROR, &err, &len);
if (ret)
goto close;
if (err) {
ret = -1;
errno = err;
perror("async rconnect");
}
}
close:
if (ret)
rs_close(rs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
/*
* Description
* Create listen fd for registering slaves.
*
*
* Return Value
* On success, RS_OK is returned. On error, RS_ERR is returned
*
*/
int rs_dump_listen(rs_master_info_t *mi)
{
int err, tries, reuseaddr;
struct sockaddr_in svr_addr;
/* init var */
reuseaddr = 1;
rs_memzero(&svr_addr, sizeof(svr_addr));
svr_addr.sin_family = AF_INET;
if(mi->listen_port <= 0) {
rs_log_err(0, "listen_port is invalid");
goto free;
}
svr_addr.sin_port = htons(mi->listen_port);
if(mi->listen_addr == NULL) {
rs_log_err(0, "listen_addr must not be null");
goto free;
}
if (inet_pton(AF_INET, mi->listen_addr, &(svr_addr.sin_addr)) != 1) {
rs_log_err(rs_errno, "inet_pton() failed, %s", mi->listen_addr);
goto free;
}
for(tries = RS_RETRY_BIND_TIMES; tries; tries--) {
mi->svr_fd = socket(AF_INET, SOCK_STREAM, 0);
if(mi->svr_fd == -1) {
rs_log_err(rs_errno, "socket() failed");
goto free;
}
if(setsockopt(mi->svr_fd, SOL_SOCKET, SO_REUSEADDR,
(const void *) &reuseaddr, sizeof(int)) == -1)
{
rs_log_err(rs_errno, "setsockopt(SO_REUSEADDR) failed, %s",
mi->listen_addr);
goto free;
}
if(bind(mi->svr_fd, (const struct sockaddr *) &svr_addr,
sizeof(svr_addr)) == -1)
{
err = errno;
rs_close(mi->svr_fd);
mi->svr_fd = -1;
if(err != EADDRINUSE) {
rs_log_err(err, "bind() failed, %s", mi->listen_addr);
goto free;
}
rs_log_info(0, "try again to bind() after %ums"
, RS_RETRY_BIND_SLEEP_MSC);
usleep(RS_RETRY_BIND_SLEEP_MSC * 1000);
continue;
}
if(listen(mi->svr_fd, RS_BACKLOG) == -1) {
rs_log_err(rs_errno, "listen() failed, %s", mi->listen_addr);
goto free;
}
break;
}
if(!tries) {
goto free;
}
return RS_OK;
free :
return RS_ERR;
}
void *rs_start_accept_thread(void *data)
{
int err, cli_fd;
socklen_t socklen;
rs_master_info_t *mi;
struct sockaddr_in cli_addr;
rs_request_dump_t *rd;
mi = (rs_master_info_t *) data;
pthread_cleanup_push(rs_free_accept_thread, mi);
if(mi == NULL) {
rs_log_err(0, "accept thread can not get master info struct");
goto free;
}
rs_memzero(&cli_addr, sizeof(cli_addr));
for( ;; ) {
cli_fd = accept(mi->svr_fd, (struct sockaddr *) &cli_addr, &socklen);
if(cli_fd == -1) {
if(rs_errno == EINTR) {
continue;
}
rs_log_err(rs_errno, "accept() failed");
goto free;
}
/* register slave */
rd = rs_get_request_dump(mi->req_dump_info);
if(rd == NULL) {
rs_log_err(0, "no more free request_dump struct");
rs_close(cli_fd);
cli_fd = -1;
continue;
}
rd->cli_fd = cli_fd;
/* init ring buffer */
if(rs_init_ring_buffer2(&(rd->ring_buf), RS_RING_BUFFER_NUM) != RS_OK)
{
goto free;
}
/* init slab */
if(rs_init_slab(&(rd->slab), NULL, mi->slab_init_size, mi->slab_factor
, mi->slab_mem_size, RS_SLAB_PREALLOC) != RS_OK)
{
goto free;
}
/* create dump thread */
if((err = pthread_create(&(rd->dump_thread),
&(mi->req_dump_info->thread_attr),
rs_start_dump_thread, (void *) rd)) != 0)
{
rs_log_err(err, "pthread_create() failed, req_dump thread");
goto free;
}
}
free:
pthread_cleanup_pop(1);
return NULL;
}
