/*---------------------------------------------------------------------------*/
static void *portal_server_cb(void *data)
{
struct hw_thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_context *ctx;
struct xio_server *server;
char str[128];
int i;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* open default event loop */
tdata->loop = xio_ev_loop_init();
/* create thread context for the client */
ctx = xio_ctx_open(NULL, tdata->loop, 0);
/* bind a listener server to a portal/url */
printf("thread [%d] - listen:%s\n", tdata->affinity, tdata->portal);
server = xio_bind(ctx, &portal_server_ops, tdata->portal, NULL, 0, tdata);
if (server == NULL)
goto cleanup;
sprintf(str,"hello world header response from thread %d",
tdata->affinity);
/* create "hello world" message */
for (i = 0; i <QUEUE_DEPTH; i++) {
tdata->rsp[i].out.header.iov_base = strdup(str);
tdata->rsp[i].out.header.iov_len =
strlen(tdata->rsp[i].out.header.iov_base);
}
/* the default xio supplied main loop */
xio_ev_loop_run(tdata->loop);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* detach the server */
xio_unbind(server);
/* free the message */
for (i = 0; i <QUEUE_DEPTH; i++)
free(tdata->rsp[i].out.header.iov_base);
cleanup:
/* free the context */
xio_ctx_close(ctx);
/* destroy the default loop */
xio_ev_loop_destroy(tdata->loop);
return NULL;
}
/*---------------------------------------------------------------------------*/
int main(int argc, char *argv[])
{
struct xio_server *server; /* server portal */
struct hw_server_data server_data;
char url[256];
struct xio_context *ctx;
void *loop;
int i;
uint16_t port = atoi(argv[2]);
memset(&server_data, 0, sizeof(server_data));
/* open default event loop */
loop = xio_ev_loop_init();
/* create thread context for the client */
ctx = xio_ctx_open(NULL, loop, 0);
/* create url to connect to */
sprintf(url, "rdma://%s:%d", argv[1], port);
/* bind a listener server to a portal/url */
server = xio_bind(ctx, &server_ops, url, NULL, 0, &server_data);
if (server == NULL)
goto cleanup;
/* spawn portals */
for (i = 0; i < MAX_THREADS; i++) {
server_data.tdata[i].affinity = i+1;
port += 1;
sprintf(server_data.tdata[i].portal, "rdma://%s:%d",
argv[1], port);
pthread_create(&server_data.tdata[i].thread_id, NULL,
portal_server_cb, &server_data.tdata[i]);
}
xio_ev_loop_run(loop);
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* join the threads */
for (i = 0; i < MAX_THREADS; i++)
pthread_join(server_data.tdata[i].thread_id, NULL);
/* free the server */
xio_unbind(server);
cleanup:
/* free the context */
xio_ctx_close(ctx);
/* destroy the default loop */
xio_ev_loop_destroy(&loop);
return 0;
}
/*---------------------------------------------------------------------------*/
static int xio_client_main(void *data)
{
char **argv = (char **) data;
struct xio_session *session;
char url[256];
struct xio_context *ctx;
struct hw_session_data *session_data;
int i = 0;
/* client session attributes */
struct xio_session_attr attr = {
&ses_ops, /* callbacks structure */
NULL, /* no need to pass the server private data */
0
};
session_data = kzalloc(sizeof(*session_data), GFP_KERNEL);
if (!session_data) {
printk("session_data alloc failed\n");
return 0;
}
/* create thread context for the client */
ctx = xio_ctx_open(XIO_LOOP_GIVEN_THREAD, NULL, current, 0, -1);
if (!ctx) {
kfree(session_data);
printk("context open filed\n");
return 0;
}
session_data->ctx = ctx;
/* create url to connect to */
sprintf(url, "rdma://%s:%s", argv[1], argv[2]);
session = xio_session_open(XIO_SESSION_REQ,
&attr, url, 0, 0, session_data);
/* connect the session */
session_data->conn = xio_connect(session, ctx, 0, NULL, session_data);
/* create "hello world" message */
for (i = 0; i < QUEUE_DEPTH; i++) {
memset(&session_data->req[i], 0, sizeof(session_data->req[i]));
session_data->req[i].out.header.iov_base =
kstrdup("hello world header request", GFP_KERNEL);
session_data->req[i].out.header.iov_len =
strlen(session_data->req[i].out.header.iov_base);
}
/* send first message */
for (i = 0; i < QUEUE_DEPTH; i++)
xio_send_request(session_data->conn, &session_data->req[i]);
/* the default xio supplied main loop */
xio_ev_loop_run(ctx);
/* normal exit phase */
printk("exit signaled\n");
/* free the message */
for (i = 0; i < QUEUE_DEPTH; i++)
kfree(session_data->req[i].out.header.iov_base);
/* free the context */
xio_ctx_close(ctx);
kfree(session_data);
printk("good bye\n");
return 0;
}
static void *worker_thread(void *data)
{
struct thread_data *tdata = data;
cpu_set_t cpuset;
struct xio_session **sessions = NULL;
struct xio_context *ctx;
struct session_data *session_data = NULL;
int j = 0, n = 0;
struct timespec start, end;
void *loop = NULL;
double *sec = NULL;
/* set affinity to thread */
CPU_ZERO(&cpuset);
CPU_SET(tdata->affinity, &cpuset);
pthread_setaffinity_np(tdata->thread_id, sizeof(cpu_set_t), &cpuset);
/* open default event loop */
loop = xio_ev_loop_init();
if (loop == NULL) {
fprintf(stderr, "Failed to allocate event loop\n");
return (void*)(-1);
}
/* create thread context for the client */
ctx= xio_ctx_open(NULL, loop, 0);
if(ctx == NULL) {
fprintf(stderr, "Failed to allocate thread context\n");
xio_ev_loop_destroy(&loop);
return (void*)-1;
}
session_data = malloc(tdata->num_sessions*sizeof(struct session_data));
sessions = malloc(tdata->num_sessions*sizeof(struct sessions *));
if( session_data == NULL || sessions == NULL ) {
fprintf(stderr, "Allocation failed\n");
xio_ctx_close(ctx);
xio_ev_loop_destroy(&loop);
if(session_data) {
free(session_data);
}
if(sessions) {
free(sessions);
}
return (void*)-1;
}
for(n = 0; n < tdata->num_sessions; n++)
{
tdata->loop = loop;
session_data[n].tdata = tdata;
}
/* client session attributes */
struct xio_session_attr attr = {
&ses_ops, /* callbacks structure */
NULL, /* no need to pass the server private data */
0
};
if(clock_gettime(CLOCK_MONOTONIC, &start)) {
fprintf(stderr, "clock_gettime() failed, errno = %d\n", errno);
}
for (j = 0; j< NUM_ITER; j++)
{
for(n = 0; n < tdata->num_sessions; n++)
{
sessions[n] = xio_session_open(XIO_SESSION_REQ,
&attr, tdata->url, 0, 0, &session_data[n]);
/* connect the session */
//fprintf(stderr, "Connect Session\n");
session_data[n].conn = xio_connect(sessions[n], ctx, 0, &session_data[n]);
}
/* the default xio supplied main loop */
xio_ev_loop_run(loop);
}
if(clock_gettime(CLOCK_MONOTONIC, &end)) {
fprintf(stderr, "clock_gettime() failed, errno = %d\n", errno);
}
/* normal exit phase */
fprintf(stdout, "exit signaled\n");
/* free the context */
xio_ctx_close(ctx);
/* destroy the default loop */
xio_ev_loop_destroy(&loop);
sec = malloc(sizeof(double));
*sec = (double)(((end.tv_sec * 1000000000 + end.tv_nsec) - (start.tv_sec * 1000000000 - start.tv_nsec))/NUM_ITER)/1000000000;
fprintf(stdout, "THREAD [ %lu ] It took %lf sec for %d sessions\n",
tdata->thread_id, *sec, tdata->num_sessions);
free(session_data);
free(sessions);
return ((void *)sec);
}