/*---------------------------------------------------------------------------*/
static int xio_server_main(void *data)
{
struct xio_server *server;
struct xio_context_params ctx_params;
init_xio_rdma_common_test();
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
test_params.ctx = xio_context_create(&ctx_params, 0, 0);
xio_assert(test_params.ctx);
session_ops.on_session_event = on_session_event;
server = xio_bind(test_params.ctx, &session_ops,
url, NULL, 0, NULL);
xio_assert(server);
pr_info("listen to %s\n", url);
xio_context_run_loop(test_params.ctx);
/* normal exit phase */
pr_info("exit signaled\n");
/* free the server */
xio_unbind(server);
xio_context_destroy(test_params.ctx);
fini_xio_rdma_common_test();
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/* Server run loop till shutdown */
void rdb_r_s_server_run(struct server_data *server_data)
{
int i;
/* default xio supplied main loop */
if (xio_context_run_loop(server_data->ctx, XIO_INFINITE) != 0) {
for (i = 0; i < server_data->nworkers; i++) {
xio_context_stop_loop(server_data->wdata[i].ctx);
}
fprintf(stderr, "failed to run event loop for server listner. reason %d - (%s)\n",
xio_errno(), xio_strerror(xio_errno()));
}
/* join the worker threads */
for (i = 0; i < server_data->nworkers; i++) {
pthread_join(server_data->wdata[i].thread_id, NULL);
}
/* free the server */
if (server_data->server)
xio_unbind(server_data->server);
/* free the context */
if (server_data->ctx)
xio_context_destroy(server_data->ctx);
xio_shutdown();
if (server_data)
free(server_data);
}
/*---------------------------------------------------------------------------*/
static void *balancer_server_cb(void *data)
{
struct xio_server *server; /* server portal */
struct server_data *server_data = data;
char url[256];
int retval = 0;
/* create thread context for the client */
server_data->ctx = xio_context_create(NULL,
server_data->user_param->poll_timeout, -1);
/* create url to connect to */
sprintf(url, "%s://*:%d",
server_data->user_param->transport,
server_data->user_param->server_port);
/* bind a listener server to a portal/url */
server = xio_bind(server_data->ctx, &server_ops, url,
NULL, 0, server_data);
if (server == NULL)
goto cleanup;
server_data->running = 1;
xio_context_run_loop(server_data->ctx, XIO_INFINITE);
/* free the server */
xio_unbind(server);
cleanup:
/* free the context */
xio_context_destroy(server_data->ctx);
pthread_exit(&retval);
}
int xio_exec_req(const struct node_id *nid, struct sd_req *hdr, void *data,
bool (*need_retry)(uint32_t epoch), uint32_t epoch,
uint32_t max_count)
{
struct xio_context *ctx = xio_context_create(NULL, 0, -1);
struct client_data cli = { .ctx = ctx };
struct xio_connection *conn = sd_xio_create_connection(ctx, nid, &cli);
struct xio_msg xreq;
struct sd_rsp rsp;
sd_assert(!is_main_thread());
memset(&rsp, 0, sizeof(rsp));
memset(&xreq, 0, sizeof(xreq));
client_msg_vec_init(&xreq);
memset(&rsp, 0, sizeof(rsp));
msg_prep_for_send(hdr, &rsp, data, &xreq);
xio_send_request(conn, &xreq);
xio_context_run_loop(ctx, XIO_INFINITE);
msg_finalize(hdr, data, cli.rsp);
xio_connection_destroy(conn);
xio_context_destroy(ctx);
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_server *server; /* server portal */
struct server_data *server_data;
char url[256];
int i;
uint16_t port = atoi(argv[2]);
server_data = calloc(1, sizeof(*server_data));
if (!server_data)
return -1;
xio_init();
/* create thread context for the client */
server_data->ctx = xio_context_create(NULL, 0);
/* create url to connect to */
sprintf(url, "rdma://%s:%d", argv[1], port);
/* bind a listener server to a portal/url */
server = xio_bind(server_data->ctx, &server_ops,
url, NULL, 0, server_data);
if (server == NULL)
goto cleanup;
/* spawn portals */
for (i = 0; i < MAX_THREADS; i++) {
server_data->tdata[i].affinity = i+1;
port += 1;
sprintf(server_data->tdata[i].portal, "rdma://%s:%d",
argv[1], port);
pthread_create(&server_data->tdata[i].thread_id, NULL,
portal_server_cb, &server_data->tdata[i]);
}
xio_context_run_loop(server_data->ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* join the threads */
for (i = 0; i < MAX_THREADS; i++)
pthread_join(server_data->tdata[i].thread_id, NULL);
/* free the server */
xio_unbind(server);
cleanup:
/* free the context */
xio_context_destroy(server_data->ctx);
xio_shutdown();
free(server_data);
return 0;
}
/*---------------------------------------------------------------------------*/
static void *portal_server_cb(void *data)
{
struct thread_data *tdata = (struct thread_data *)data;
cpu_set_t cpuset;
struct xio_server *server;
int retval = 0;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* prepare data for the cuurent thread */
tdata->pool = msg_pool_alloc(MAX_POOL_SIZE, 0, 1);
if (tdata->pool == NULL) {
retval = -1;
goto exit;
}
/* create thread context for the client */
tdata->ctx = xio_context_create(NULL, test_config.poll_timeout,
tdata->affinity);
/* bind a listener server to a portal/url */
printf("thread [%d] - listen:%s\n", tdata->affinity, tdata->portal);
server = xio_bind(tdata->ctx, &portal_server_ops, tdata->portal,
NULL, 0, tdata);
if (server == NULL) {
printf("**** Error - xio_bind failed. %s\n",
xio_strerror(xio_errno()));
retval = -1;
goto cleanup;
}
/* the default xio supplied main loop */
xio_context_run_loop(tdata->ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "thread [%d] - exit signaled\n", tdata->affinity);
/* detach the server */
xio_unbind(server);
if (tdata->pool)
msg_pool_free(tdata->pool);
if (tdata->reg_mem.addr)
xio_mem_free(&tdata->reg_mem);
cleanup:
/* free the context */
xio_context_destroy(tdata->ctx);
exit:
pthread_exit((void *)(unsigned long)retval);
}
/*---------------------------------------------------------------------------*/
static void *portal_server_cb(void *data)
{
struct thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_server *server;
int retval = 0;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* prepare data for the cuurent thread */
tdata->pool = msg_pool_alloc(tdata->user_param->queue_depth + 64);
/* create thread context for the client */
tdata->ctx = xio_context_create(NULL, tdata->user_param->poll_timeout,
tdata->affinity);
/* bind a listener server to a portal/url */
server = xio_bind(tdata->ctx, &portal_server_ops,
tdata->user_param->portals_arr[tdata->portal_index],
NULL, 0, tdata);
if (server == NULL)
goto cleanup;
/* the default xio supplied main loop */
xio_context_run_loop(tdata->ctx, XIO_INFINITE);
/* detach the server */
xio_unbind(server);
if (tdata->pool)
msg_pool_free(tdata->pool);
if (tdata->mr)
xio_dereg_mr(&tdata->mr);
if (tdata->buf)
free(tdata->buf);
if (tdata->out_xbuf)
xio_free(&tdata->out_xbuf);
if (tdata->in_xbuf)
xio_free(&tdata->in_xbuf);
cleanup:
/* free the context */
xio_context_destroy(tdata->ctx);
pthread_exit(&retval);
}
/* Server I/O manager */
void *rdb_r_s_io_worker(void *data)
{
struct io_worker_data *worker_data;
struct xio_server *server = NULL;
cpu_set_t cpuset;
worker_data = (struct io_worker_data *) data;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(worker_data->affinity, &cpuset);
pthread_setaffinity_np(worker_data->thread_id, sizeof(cpu_set_t), &cpuset);
worker_data->ctx = xio_context_create(NULL, POLLING_TIMEOUT,
worker_data->affinity);
if (!worker_data->ctx) {
fprintf(stderr, "failed to create context for thread %s. reason %d - (%s)\n",
worker_data->portal, xio_errno(), xio_strerror(xio_errno()));
goto out;
}
server = xio_bind(worker_data->ctx, &io_worker_ops,
worker_data->portal, NULL, 0, worker_data);
if (!server) {
fprintf(stderr, "failed to bind context for thread %s. reason %d - (%s)\n",
worker_data->portal, xio_errno(), xio_strerror(xio_errno()));
goto out;
}
xio_mem_alloc(MAX_VALUE_SIZE, &worker_data->reg_mem);
if (!worker_data->reg_mem.addr) {
fprintf(stderr, "failed to allocate accelio transfer buffer for thread %s. reason %d - (%s)\n",
worker_data->portal, xio_errno(), xio_strerror(xio_errno()));
goto out;
}
/* the default xio supplied main loop */
if (xio_context_run_loop(worker_data->ctx, XIO_INFINITE) != 0) {
fprintf(stderr, "failed to run event loop for thread %s. reason %d - (%s)\n",
worker_data->portal, xio_errno(), xio_strerror(xio_errno()));
}
out:
/* detach the worker */
if (server)
xio_unbind(server);
if (worker_data->reg_mem.addr)
xio_mem_free(&worker_data->reg_mem);
/* free the context */
if (worker_data->ctx)
xio_context_destroy(worker_data->ctx);
return NULL;
}
/*---------------------------------------------------------------------------*/
static void *portal_server_cb(void *data)
{
struct thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_server *server;
char str[128];
int i;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* create thread context for the client */
tdata->ctx = xio_context_create(NULL, 0);
/* bind a listener server to a portal/url */
printf("thread [%d] - listen:%s\n", tdata->affinity, tdata->portal);
server = xio_bind(tdata->ctx, &portal_server_ops, tdata->portal,
NULL, 0, tdata);
if (server == NULL)
goto cleanup;
sprintf(str, "hello world header response from thread %d",
tdata->affinity);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
tdata->rsp[i].out.header.iov_base = strdup(str);
tdata->rsp[i].out.header.iov_len =
strlen(tdata->rsp[i].out.header.iov_base);
}
/* the default xio supplied main loop */
xio_context_run_loop(tdata->ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* detach the server */
xio_unbind(server);
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++)
free(tdata->rsp[i].out.header.iov_base);
cleanup:
/* free the context */
xio_context_destroy(tdata->ctx);
return NULL;
}
/*---------------------------------------------------------------------------*/
static void *portal_server_cb(void *data)
{
struct thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_server *server;
int retval = 0;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* prepare data for the cuurent thread */
tdata->pool = msg_pool_alloc(MAX_POOL_SIZE,
test_config.hdr_len, test_config.data_len,
0, 0);
/* create thread context for the client */
tdata->ctx = xio_context_create(NULL, test_config.poll_timeout);
/* bind a listener server to a portal/url */
printf("thread [%d] - listen:%s\n", tdata->affinity, tdata->portal);
server = xio_bind(tdata->ctx, &portal_server_ops, tdata->portal,
NULL, 0, tdata);
if (server == NULL)
goto cleanup;
/* the default xio supplied main loop */
xio_context_run_loop(tdata->ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "thread [%d] - exit signaled\n", tdata->affinity);
/* detach the server */
xio_unbind(server);
if (tdata->pool)
msg_pool_free(tdata->pool);
if (tdata->mr)
xio_dereg_mr(&tdata->mr);
if (tdata->buf)
free(tdata->buf);
cleanup:
/* free the context */
xio_context_destroy(tdata->ctx);
pthread_exit(&retval);
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_session *session;
char url[256];
struct session_data session_data;
int i = 0;
struct xio_session_params params;
if (argc < 3) {
printf("Usage: %s <host> <port> <transport:optional>\n",
argv[0]);
exit(1);
}
memset(&session_data, 0, sizeof(session_data));
memset(¶ms, 0, sizeof(params));
/* initialize library */
xio_init();
/* create thread context for the client */
session_data.ctx = xio_context_create(NULL, 0, -1);
/* create url to connect to */
if (argc > 3)
sprintf(url, "%s://%s:%s", argv[3], argv[1], argv[2]);
else
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
params.type = XIO_SESSION_CLIENT;
params.ses_ops = &ses_ops;
params.user_context = &session_data;
params.uri = url;
session = xio_session_create(¶ms);
/* connect the session */
session_data.conn = xio_connect(session, session_data.ctx,
0, NULL, &session_data);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
memset(&session_data.req[i], 0, sizeof(session_data.req[i]));
/* header */
session_data.req[i].out.header.iov_base =
strdup("hello world header request");
session_data.req[i].out.header.iov_len =
strlen(session_data.req[i].out.header.iov_base) + 1;
/* iovec[0]*/
session_data.req[i].in.sgl_type = XIO_SGL_TYPE_IOV;
session_data.req[i].in.data_iov.max_nents = XIO_IOVLEN;
session_data.req[i].out.sgl_type = XIO_SGL_TYPE_IOV;
session_data.req[i].out.data_iov.max_nents = XIO_IOVLEN;
session_data.req[i].out.data_iov.sglist[0].iov_base =
strdup("hello world data request");
session_data.req[i].out.data_iov.sglist[0].iov_len =
strlen(session_data.req[i].out.data_iov.sglist[0].iov_base) + 1;
session_data.req[i].out.data_iov.nents = 1;
}
/* send first message */
for (i = 0; i < QUEUE_DEPTH; i++) {
xio_send_request(session_data.conn, &session_data.req[i]);
session_data.nsent++;
}
/* event dispatcher is now running */
xio_context_run_loop(session_data.ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++) {
free(session_data.req[i].out.header.iov_base);
free(session_data.req[i].out.data_iov.sglist[0].iov_base);
}
/* free the context */
xio_context_destroy(session_data.ctx);
xio_shutdown();
printf("good bye\n");
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *const argv[])
{
struct xio_session *session;
char url[256];
int i;
struct sigaction sa;
int c;
char *addr = NULL;
char *port = NULL;
char *trans = NULL;
struct xio_session_params params;
struct xio_connection_params cparams;
while (1) {
c = getopt_long(argc, argv, "a:p:r:hdnV", longopts, NULL);
if (c == -1)
break;
switch (c) {
case 'a':
addr = optarg;
break;
case 'p':
port = optarg;
break;
case 'r':
trans = optarg;
break;
case 'h':
usage(argv[0], 0);
case 'd':
debug_flag++;
nofork_flag++;
break;
case 'n':
nofork_flag++;
break;
case 'V':
printf("%s\n", PACKAGE_STRING);
exit(0);
break;
default:
usage(argv[0], 1);
break;
}
}
memset(&sa, 0, sizeof(sa));
sa.sa_handler = signal_handler;
sigaction(SIGINT, &sa, NULL);
sigaction(SIGTERM, &sa, NULL);
sigaction(SIGQUIT, &sa, NULL);
sigaction(SIGHUP, &sa, NULL);
sa.sa_handler = SIG_IGN;
sa.sa_flags = SA_RESTART;
sigaction(SIGPIPE, &sa, NULL);
if (!nofork_flag && daemon(0, 0)) {
logerr("daemon() failed");
exit(1);
}
if (!debug_flag) {
openlog("xioclntd", LOG_PID, LOG_DAEMON);
use_syslog = 1;
}
/* Create the process PID file */
if (create_pidfile(pid_file) != 0)
exit(EXIT_FAILURE);
memset(&session_data, 0, sizeof(session_data));
memset(¶ms, 0, sizeof(params));
memset(&cparams, 0, sizeof(cparams));
/* initialize library */
xio_init();
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
memset(&session_data.req[i], 0, sizeof(session_data.req[i]));
/* header */
session_data.req[i].out.header.iov_base =
strdup("hello world header request");
session_data.req[i].out.header.iov_len =
strlen((const char *)
session_data.req[i].out.header.iov_base) + 1;
/* iovec[0]*/
session_data.req[i].out.sgl_type = XIO_SGL_TYPE_IOV;
session_data.req[i].out.data_iov.max_nents = XIO_IOVLEN;
session_data.req[i].out.data_iov.sglist[0].iov_base =
strdup("hello world iovec request");
session_data.req[i].out.data_iov.sglist[0].iov_len =
strlen((const char *)
session_data.req[i].out.data_iov.sglist[0].iov_base) + 1;
session_data.req[i].out.data_iov.nents = 1;
}
/* create thread context for the client */
session_data.ctx = xio_context_create(NULL, 0, -1);
/* create url to connect to */
if (trans)
sprintf(url, "%s://%s:%s", trans, addr, port);
else
sprintf(url, "rdma://%s:%s", addr, port);
params.type = XIO_SESSION_CLIENT;
params.ses_ops = &ses_ops;
params.user_context = &session_data;
params.uri = url;
reconnect:
session = xio_session_create(¶ms);
cparams.session = session;
cparams.ctx = session_data.ctx;
cparams.conn_user_context = &session_data;
/* connect the session */
session_data.conn = xio_connect(&cparams);
/* event dispatcher is now running */
xio_context_run_loop(session_data.ctx, XIO_INFINITE);
if (reconnect_flag || reload_flag) {
session_data.cnt = 0;
if (reconnect_flag)
sleep(1);
reload_flag = 0;
reconnect_flag = 0;
goto reconnect;
}
/* normal exit phase */
logit(LOG_INFO, "exit signaled\n");
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++) {
free(session_data.req[i].out.header.iov_base);
free(session_data.req[i].out.data_iov.sglist[0].iov_base);
}
/* free the context */
xio_context_destroy(session_data.ctx);
xio_shutdown();
remove_pidfile();
if (use_syslog)
closelog();
return 0;
}
/*---------------------------------------------------------------------------*/
static void *worker_thread(void *data)
{
struct thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_msg *msg;
int i;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* prepare data for the cuurent thread */
tdata->pool = msg_pool_alloc(tdata->user_param->queue_depth);
/* create thread context for the client */
tdata->ctx = xio_context_create(NULL, tdata->user_param->poll_timeout,
tdata->affinity);
/* connect the session */
tdata->conn = xio_connect(tdata->session, tdata->ctx,
tdata->cid, NULL, tdata);
if (tdata->data_len)
tdata->xbuf = xio_alloc(tdata->data_len);
for (i = 0; i < tdata->user_param->queue_depth; i++) {
/* create transaction */
msg = msg_pool_get(tdata->pool);
if (msg == NULL)
break;
/* get pointers to internal buffers */
msg->in.header.iov_len = 0;
msg->in.data_iovlen = 0;
msg->out.header.iov_len = 0;
if (tdata->data_len) {
msg->out.data_iovlen = 1;
msg->out.data_iov[0].iov_base = tdata->xbuf->addr;
msg->out.data_iov[0].iov_len = tdata->xbuf->length;
msg->out.data_iov[0].mr = tdata->xbuf->mr;
} else {
msg->out.data_iovlen = 0;
}
msg->user_context = (void *)get_cycles();
/* send first message */
if (xio_send_request(tdata->conn, msg) == -1) {
if (xio_errno() != EAGAIN)
printf("**** [%p] Error - xio_send_request " \
"failed. %s\n",
tdata->session,
xio_strerror(xio_errno()));
msg_pool_put(tdata->pool, msg);
return 0;
}
if (tdata->do_stat)
tdata->stat.scnt++;
tdata->tx_nr++;
}
/* the default xio supplied main loop */
xio_context_run_loop(tdata->ctx, XIO_INFINITE);
/* normal exit phase */
if (tdata->pool)
msg_pool_free(tdata->pool);
if (tdata->xbuf)
xio_free(&tdata->xbuf);
/* free the context */
xio_context_destroy(tdata->ctx);
return NULL;
}
/*---------------------------------------------------------------------------*/
static int xio_client_main(void *data)
{
char **argv = (char **)data;
struct xio_session *session;
struct xio_session_params params;
struct xio_context_params ctx_params;
struct xio_connection_params cparams;
char url[256];
struct xio_context *ctx;
struct session_data *session_data;
int i = 0;
atomic_add(2, &module_state);
session_data = vzalloc(sizeof(*session_data));
if (!session_data) {
/*pr_err("session_data alloc failed\n");*/
return 0;
}
/* create thread context for the client */
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
ctx = xio_context_create(&ctx_params, 0, -1);
if (!ctx) {
vfree(session_data);
pr_err("context open filed\n");
return 0;
}
session_data->ctx = ctx;
/* create url to connect to */
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
memset(¶ms, 0, sizeof(params));
params.type = XIO_SESSION_CLIENT;
params.ses_ops = &ses_ops;
params.user_context = session_data;
params.uri = url;
session = xio_session_create(¶ms);
memset(&cparams, 0, sizeof(cparams));
cparams.session = session;
cparams.ctx = ctx;
cparams.conn_user_context = session_data;
/* connect the session */
session_data->session = session;
session_data->connection = xio_connect(&cparams);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
struct xio_vmsg *omsg;
void *buf;
omsg = &session_data->req[i].out;
memset(&session_data->req[i], 0, sizeof(session_data->req[i]));
/* header */
buf = kstrdup("hello world header request", GFP_KERNEL);
session_data->req[i].out.header.iov_base = buf;
session_data->req[i].out.header.iov_len = strlen(buf) + 1;
/* iovec[0]*/
omsg->sgl_type = XIO_SGL_TYPE_SCATTERLIST;
sg_alloc_table(&omsg->data_tbl, SG_TBL_LEN, GFP_KERNEL);
/* currently only one entry */
buf = kstrdup("hello world iovec request", GFP_KERNEL);
sg_init_one(omsg->data_tbl.sgl, buf, strlen(buf) + 1);
/* orig_nents is XIO_IOVLEN */
omsg->data_tbl.nents = 1;
}
/* send first message */
for (i = 0; i < QUEUE_DEPTH; i++)
xio_send_request(session_data->connection,
&session_data->req[i]);
g_session_data = session_data;
/* the default xio supplied main loop */
if (atomic_add_unless(&module_state, 4, 0x83))
xio_context_run_loop(ctx);
atomic_sub(4, &module_state);
/* normal exit phase */
pr_info("exit signaled\n");
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++) {
kfree(session_data->req[i].out.header.iov_base);
/* Currently need to release only one entry */
kfree(sg_virt(session_data->req[i].out.data_tbl.sgl));
sg_free_table(&session_data->req[i].out.data_tbl);
}
/* free the context */
xio_context_destroy(ctx);
vfree(session_data);
pr_info("good bye\n");
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_client_main(void *data)
{
char url[256];
struct xio_session_params params;
struct xio_context_params ctx_params;
struct xio_connection_params cparams;
int error;
int retval = 0;
atomic_add(2, &module_state);
print_counter = PRINT_COUNTER;
print_test_config(&test_config);
memset(¶ms, 0, sizeof(params));
memset(&cparams, 0, sizeof(cparams));
/* prepare buffers for this test */
if (msg_api_init(&msg_params,
test_config.hdr_len, test_config.data_len, 0) != 0) {
pr_err("msg_api_init failed\n");
return -1;
}
pool = msg_pool_alloc(MAX_POOL_SIZE, 1, 1);
if (!pool) {
pr_err("msg_pool_alloc failed\n");
goto cleanup;
}
/* create thread context for the server */
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
ctx = xio_context_create(&ctx_params,
POLLING_TIMEOUT, cpu);
if (!ctx) {
pr_err("context open failed\n");
goto cleanup;
}
sprintf(url, "%s://%s:%d",
test_config.transport,
test_config.server_addr,
test_config.server_port);
params.type = XIO_SESSION_CLIENT;
params.ses_ops = &ses_ops;
params.uri = url;
g_session = xio_session_create(¶ms);
if (!g_session)
pr_err("session creation failed\n");
cparams.session = g_session;
cparams.ctx = ctx;
cparams.conn_idx = test_config.conn_idx;
/* connect the session */
g_connection = xio_connect(&cparams);
/* the default xio supplied main loop */
if (atomic_add_unless(&module_state, 4, 0x83))
retval = xio_context_run_loop(ctx);
atomic_sub(4, &module_state);
if (retval != 0) {
error = xio_errno();
pr_err("running event loop failed. reason %d - (%s)\n",
error, xio_strerror(error));
xio_assert(retval == 0);
}
/* normal exit phase */
pr_info("exit signaled\n");
xio_context_destroy(ctx);
msg_pool_free(pool);
cleanup:
msg_api_free(&msg_params);
pr_info("exit complete\n");
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_server_main(void *data)
{
struct xio_server *server;
struct xio_context_params ctx_params;
char url[256];
atomic_add(2, &module_state);
print_test_config(&test_config);
g_test_params.finite_run = test_config.finite_run;
g_test_params.disconnect_nr = PRINT_COUNTER * DISCONNECT_FACTOR;
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
g_test_params.ctx = xio_context_create(&ctx_params,
0, g_test_params.cpu);
if (!g_test_params.ctx) {
int error = xio_errno();
pr_err("context creation failed. reason %d - (%s)\n",
error, xio_strerror(error));
goto cleanup;
}
sprintf(url, "%s://%s:%d",
test_config.transport,
test_config.server_addr,
test_config.server_port);
server = xio_bind(g_test_params.ctx, &server_ops,
url, NULL, 0, &g_test_params);
if (server) {
pr_info("listen to %s\n", url);
if (atomic_add_unless(&module_state, 4, 0x83))
xio_context_run_loop(g_test_params.ctx);
atomic_sub(4, &module_state);
/* normal exit phase */
pr_info("exit signaled\n");
/* free the server */
xio_unbind(server);
} else {
pr_err("**** Error - xio_bind failed - %s. " \
"Did you load a transport module?\n",
xio_strerror(xio_errno()));
/*xio_assert(0);*/
}
xio_context_destroy(g_test_params.ctx);
kfree(g_test_params.xbuf);
g_test_params.xbuf = NULL;
cleanup:
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_server_main(void *data)
{
char **argv = (char **)data;
struct xio_server *server; /* server portal */
struct server_data *server_data;
char url[256];
struct xio_context_params ctx_params;
struct xio_context *ctx;
int i;
atomic_add(2, &module_state);
server_data = vzalloc(sizeof(*server_data));
if (!server_data) {
/*pr_err("server_data alloc failed\n");*/
return 0;
}
/* create thread context for the server */
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
ctx = xio_context_create(&ctx_params, 0, -1);
if (!ctx) {
vfree(server_data);
pr_err("context open filed\n");
return 0;
}
server_data->ctx = ctx;
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
server_data->rsp[i].out.header.iov_base =
kstrdup("hello world header response", GFP_KERNEL);
server_data->rsp[i].out.header.iov_len =
strlen(server_data->rsp[i].out.header.iov_base) + 1;
server_data->rsp[i].out.sgl_type = XIO_SGL_TYPE_SCATTERLIST;
server_data->rsp[i].out.data_tbl.orig_nents = 0;
server_data->rsp[i].out.data_tbl.nents = 0;
server_data->rsp[i].out.data_tbl.sgl = NULL;
}
/* create url to connect to */
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
/* bind a listener server to a portal/url */
server = xio_bind(ctx, &server_ops, url, NULL, 0, server_data);
if (server) {
pr_info("listen to %s\n", url);
g_server_data = server_data;
if (atomic_add_unless(&module_state, 4, 0x83))
xio_context_run_loop(ctx);
atomic_sub(4, &module_state);
/* normal exit phase */
pr_info("exit signaled\n");
/* free the server */
xio_unbind(server);
}
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++)
kfree(server_data->rsp[i].out.header.iov_base);
/* free the context */
xio_context_destroy(ctx);
vfree(server_data);
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *const argv[])
{
struct xio_server *server; /* server portal */
char url[256];
int i;
struct sigaction sa;
int c;
char *addr = NULL;
char *port = NULL;
char *trans = NULL;
while (1) {
c = getopt_long(argc, argv, "a:p:r:hdnV", longopts, NULL);
if (c == -1)
break;
switch (c) {
case 'a':
addr = optarg;
break;
case 'p':
port = optarg;
break;
case 'r':
trans = optarg;
break;
case 'h':
usage(argv[0], 0);
case 'd':
debug_flag++;
nofork_flag++;
break;
case 'n':
nofork_flag++;
break;
case 'V':
printf("%s\n", PACKAGE_STRING);
exit(0);
break;
default:
usage(argv[0], 1);
break;
}
}
memset(&sa, 0, sizeof(sa));
sa.sa_handler = signal_handler;
sigaction(SIGINT, &sa, NULL);
sigaction(SIGTERM, &sa, NULL);
sigaction(SIGQUIT, &sa, NULL);
sigaction(SIGHUP, &sa, NULL);
sa.sa_handler = SIG_IGN;
sa.sa_flags = SA_RESTART;
sigaction(SIGPIPE, &sa, NULL);
if (!nofork_flag && daemon(0, 0)) {
logerr("daemon() failed");
exit(1);
}
if (!debug_flag) {
openlog("xiosrvd", LOG_PID, LOG_DAEMON);
use_syslog = 1;
}
/* Create the process PID file */
if (create_pidfile(pid_file) != 0)
exit(EXIT_FAILURE);
/* initialize library */
xio_init();
/* create "hello world" message */
memset(&server_data, 0, sizeof(server_data));
for (i = 0; i < QUEUE_DEPTH; i++) {
server_data.rsp[i].out.header.iov_base =
strdup("hello world header response");
server_data.rsp[i].out.header.iov_len =
strlen((const char *)
server_data.rsp[i].out.header.iov_base) + 1;
}
/* create thread context for the client */
server_data.ctx = xio_context_create(NULL, 0, -1);
/* create url to connect to */
if (trans)
sprintf(url, "%s://%s:%s", trans, addr, port);
else
sprintf(url, "rdma://%s:%s", addr, port);
reload:
/* bind a listener server to a portal/url */
server = xio_bind(server_data.ctx, &server_ops,
url, NULL, 0, &server_data);
if (server) {
logit(LOG_INFO, "listen to %s", url);
xio_context_run_loop(server_data.ctx, XIO_INFINITE);
/* free the server */
xio_unbind(server);
if (reload_flag) {
reload_flag = 0;
goto reload;
}
/* normal exit phase */
logit(LOG_INFO, "exit signaled");
}
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++)
free(server_data.rsp[i].out.header.iov_base);
/* free the context */
xio_context_destroy(server_data.ctx);
xio_shutdown();
remove_pidfile();
if (use_syslog)
closelog();
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_session *session;
char url[256];
struct session_data session_data;
int i = 0;
struct event timeout;
struct event xio_event;
struct timeval tv;
struct xio_poll_params poll_params;
struct xio_session_params params;
if (argc < 3) {
printf("Usage: %s <host> <port> <transport:optional>\n",
argv[0]);
exit(1);
}
memset(&session_data, 0, sizeof(session_data));
memset(¶ms, 0, sizeof(params));
/* initialize library */
xio_init();
/* create thread context for the client */
session_data.ctx = xio_context_create(NULL, 0, -1);
/* get poll parameters for libevent */
xio_context_get_poll_params(session_data.ctx, &poll_params);
/* create url to connect to */
if (argc > 3)
sprintf(url, "%s://%s:%s", argv[3], argv[1], argv[2]);
else
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
params.type = XIO_SESSION_CLIENT;
params.ses_ops = &ses_ops;
params.user_context = &session_data;
params.uri = url;
session = xio_session_create(¶ms);
/* connect the session */
session_data.conn = xio_connect(session, session_data.ctx,
0, NULL, &session_data);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
memset(&session_data.req[i], 0, sizeof(session_data.req[i]));
/* header */
session_data.req[i].out.header.iov_base =
strdup("hello world header request");
session_data.req[i].out.header.iov_len =
strlen(session_data.req[i].out.header.iov_base) + 1;
/* iovec[0]*/
session_data.req[i].out.sgl_type = XIO_SGL_TYPE_IOV;
session_data.req[i].out.data_iov.max_nents = XIO_IOVLEN;
session_data.req[i].out.data_iov.sglist[0].iov_base =
strdup("hello world iovec request");
session_data.req[i].out.data_iov.sglist[0].iov_len =
strlen(session_data.req[i].out.data_iov.sglist[0].iov_base) + 1;
session_data.req[i].out.data_iov.nents = 1;
}
/* send first message */
for (i = 0; i < QUEUE_DEPTH; i++) {
xio_send_request(session_data.conn, &session_data.req[i]);
session_data.nsent++;
}
/* Initialize the event library */
session_data.evbase = event_base_new();
/* Initialize one timer event */
event_assign(&timeout, session_data.evbase, -1,
EV_PERSIST, timeout_cb, (void *)&timeout);
evutil_timerclear(&tv);
tv.tv_sec = 2;
event_add(&timeout, &tv);
event_assign(&xio_event, session_data.evbase, poll_params.fd,
EV_READ|EV_PERSIST, xio_event_handler,
(void *)&poll_params);
/* Add it to the active events, without a timeout */
event_add(&xio_event, NULL);
event_base_dispatch(session_data.evbase);
fprintf(stdout, "exit signaled\n");
event_base_free(session_data.evbase);
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++) {
free(session_data.req[i].out.header.iov_base);
free(session_data.req[i].out.data_iov.sglist[0].iov_base);
}
/* free the context */
xio_context_destroy(session_data.ctx);
xio_shutdown();
printf("good bye\n");
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_server *server; /* server portal */
struct server_data server_data;
char url[256];
struct xio_msg *rsp;
int i, opt, optlen;
struct xio_reg_mem xbuf;
int msg_size = 0;
int max_msg_size = 0;
uint8_t *data = NULL;
xbuf.addr = NULL;
xbuf.mr = NULL;
if (argc < 3) {
printf("Usage: %s <host> <port> <transport:optional>"\
"<finite run:optional> <msg size:optional>\n", argv[0]);
exit(1);
}
if (argc > 4)
test_disconnect = atoi(argv[4]);
else
test_disconnect = 0;
if (argc > 5)
msg_size = atoi(argv[5]);
/* initialize library */
xio_init();
/* get max msg size */
/* this size distinguishes between big and small msgs, where for small msgs rdma_post_send/rdma_post_recv
are called as opposed to to big msgs where rdma_write/rdma_read are called */
xio_get_opt(NULL, XIO_OPTLEVEL_ACCELIO,
XIO_OPTNAME_MAX_INLINE_XIO_DATA,
&opt, &optlen);
max_msg_size = opt;
/* create "hello world" message */
memset(&server_data, 0, sizeof(server_data));
rsp = server_data.rsp_ring;
for (i = 0; i < QUEUE_DEPTH; i++) {
/* header */
rsp->out.header.iov_base =
strdup("hello world header response");
rsp->out.header.iov_len =
strlen((const char *)
rsp->out.header.iov_base) + 1;
rsp->out.sgl_type = XIO_SGL_TYPE_IOV;
rsp->out.data_iov.max_nents = XIO_IOVLEN;
/* data */
if (msg_size < max_msg_size) { /* small msgs */
rsp->out.data_iov.sglist[0].iov_base =
strdup("hello world data response");
} else { /* big msgs */
if (data == NULL) {
printf("allocating xio memory...\n");
xio_mem_alloc(msg_size, &xbuf);
if (xbuf.addr != NULL){
data = (uint8_t *)xbuf.addr;
memset(data, 0, msg_size);
sprintf((char *)data, "hello world data response");
} else {
printf("ERROR - xio_mem_alloc failed.\n");
exit(1);
}
}
rsp->out.data_iov.sglist[0].mr = xbuf.mr;
rsp->out.data_iov.sglist[0].iov_base = data;
}
rsp->out.data_iov.sglist[0].iov_len =
strlen((const char *)
rsp->out.data_iov.sglist[0].iov_base) + 1;
rsp->out.data_iov.nents = 1;
rsp++;
}
/* create thread context for the client */
server_data.ctx = xio_context_create(NULL, 0, -1);
/* create url to connect to */
if (argc > 3)
sprintf(url, "%s://%s:%s", argv[3], argv[1], argv[2]);
else
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
/* bind a listener server to a portal/url */
server = xio_bind(server_data.ctx, &server_ops,
url, NULL, 0, &server_data);
if (server) {
printf("listen to %s\n", url);
xio_context_run_loop(server_data.ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* free the server */
xio_unbind(server);
}
/* free the message */
rsp = server_data.rsp_ring;
for (i = 0; i < QUEUE_DEPTH; i++) {
free(rsp->out.header.iov_base);
if (msg_size < max_msg_size) free(rsp->out.data_iov.sglist[0].iov_base);
rsp++;
}
if (data) {
printf("freeing xio memory...\n");
xio_mem_free(&xbuf);
xbuf.addr = NULL;
}
/* free the context */
xio_context_destroy(server_data.ctx);
xio_shutdown();
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_server_main(void *data)
{
char **argv = (char **)data;
struct xio_server *server; /* server portal */
struct server_data *server_data;
char url[256];
struct xio_context_params ctx_params;
struct xio_context *ctx;
int i;
struct xio_vmsg *omsg;
void *buf = NULL;
unsigned long data_len = 0;
atomic_add(2, &module_state);
server_data = vzalloc(sizeof(*server_data));
if (!server_data) {
/*pr_err("server_data alloc failed\n");*/
return 0;
}
/* create thread context for the server */
memset(&ctx_params, 0, sizeof(ctx_params));
ctx_params.flags = XIO_LOOP_GIVEN_THREAD;
ctx_params.worker = current;
ctx = xio_context_create(&ctx_params, 0, -1);
if (!ctx) {
vfree(server_data);
pr_err("context open filed\n");
return 0;
}
server_data->ctx = ctx;
/* create "hello world" message */
if (argv[3] != NULL && kstrtoul(argv[3], 0, &data_len)) { /* check, convert and assign data_len */
data_len = 0;
}
for (i = 0; i < QUEUE_DEPTH; i++) {
xio_reinit_msg(&server_data->rsp[i]);
omsg = &server_data->rsp[i].out;
/* header */
server_data->rsp[i].out.header.iov_base = kstrdup("hello world header rsp 1", GFP_KERNEL);
server_data->rsp[i].out.header.iov_len = strlen((const char *) server_data->rsp[i].out.header.iov_base) + 1;
/* iovec[0]*/
sg_alloc_table(&omsg->data_tbl, 64, GFP_KERNEL);
/* currently only one entry */
xio_init_vmsg(omsg, 1); /* one entry (max_nents) */
if (data_len < max_data_len) { /* small msgs */
buf = kstrdup("hello world iovec rsp", GFP_KERNEL);
} else { /* big msgs */
if (!buf) {
pr_info("allocating xio memory...\n");
buf = kmalloc(data_len, GFP_KERNEL);
memcpy(buf, "hello world iovec rsp", 22);
}
}
sg_init_one(omsg->data_tbl.sgl, buf, strlen(buf) + 1);
/* orig_nents is 64 */
vmsg_sglist_set_nents(omsg, 1);
}
/* create url to connect to */
sprintf(url, "%s://%s:%s", argv[4], argv[1], argv[2]);
/* bind a listener server to a portal/url */
server = xio_bind(ctx, &server_ops, url, NULL, 0, server_data);
if (server) {
pr_info("listen to %s\n", url);
g_server_data = server_data;
if (atomic_add_unless(&module_state, 4, 0x83))
xio_context_run_loop(ctx);
atomic_sub(4, &module_state);
/* normal exit phase */
pr_info("exit signaled\n");
/* free the server */
xio_unbind(server);
}
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++) {
kfree(server_data->rsp[i].out.header.iov_base);
if (data_len < max_data_len) {
/* Currently need to release only one entry */
kfree(sg_virt(server_data->rsp[i].out.data_tbl.sgl));
}
xio_fini_vmsg(&server_data->rsp[i].out);
}
if (buf){
pr_info("freeing xio memory...\n");
kfree(buf);
buf = NULL;
}
/* free the context */
xio_context_destroy(ctx);
vfree(server_data);
complete_and_exit(&cleanup_complete, 0);
return 0;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_session *session;
char url[256];
struct session_data session_data;
int i = 0;
/* client session attributes */
struct xio_session_attr attr = {
&ses_ops, /* callbacks structure */
NULL, /* no need to pass the server private data */
0
};
memset(&session_data, 0, sizeof(session_data));
/* initialize library */
xio_init();
/* create thread context for the client */
session_data.ctx = xio_context_create(NULL, 0);
/* create url to connect to */
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
session = xio_session_create(XIO_SESSION_CLIENT,
&attr, url, 0, 0, &session_data);
/* connect the session */
session_data.conn = xio_connect(session, session_data.ctx,
0, NULL, &session_data);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
memset(&session_data.req[i], 0, sizeof(session_data.req[i]));
session_data.req[i].out.header.iov_base =
strdup("hello world header request");
session_data.req[i].out.header.iov_len =
strlen(session_data.req[i].out.header.iov_base) + 1;
}
/* send first message */
for (i = 0; i < QUEUE_DEPTH; i++) {
xio_send_request(session_data.conn, &session_data.req[i]);
session_data.nsent++;
}
/* event dispatcher is now running */
xio_context_run_loop(session_data.ctx, XIO_INFINITE);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++)
free(session_data.req[i].out.header.iov_base);
/* free the context */
xio_context_destroy(session_data.ctx);
xio_shutdown();
printf("good bye\n");
return 0;
}