/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct server_data *server_data = (struct server_data *)cb_user_context; printf("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: server_data->connection = event_data->conn; break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); server_data->connection = NULL; break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); xio_context_stop_loop(server_data->ctx); /* exit */ break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct server_data *server_data = cb_user_context; int i; printf("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); for (i = 0; i < MAX_THREADS; i++) xio_context_stop_loop(server_data->tdata[i].ctx, 0); xio_context_stop_loop(server_data->ctx, 0); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct session_data *session_data = (struct session_data *) cb_user_context; logit(LOG_INFO, "session event: %s. reason: %s\n", xio_session_event_str(event_data->event), xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_CONNECTION_DISCONNECTED_EVENT: case XIO_SESSION_CONNECTION_REFUSED_EVENT: reconnect_flag = 1; break; case XIO_SESSION_CONNECTION_ESTABLISHED_EVENT: on_connection_established(event_data->conn, session_data); break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); session_data->conn = NULL; break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); xio_context_stop_loop(session_data->ctx); /* exit */ break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { /*struct xio_connection_attr conn_attr;*/ pr_info("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: if (!test_params.connection) test_params.connection = event_data->conn; break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: if (event_data->reason != XIO_E_SESSION_REJECTED) test_params.connection = NULL; xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); if (event_data->reason != XIO_E_SESSION_REJECTED) xio_context_stop_loop(test_params.ctx); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_prv_data) { printf("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: process_request(NULL); xio_session_destroy(session); if (test_config.finite_run) xio_context_stop_loop(ctx); /* exit */ break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct server_data *server_data = cb_user_context; pr_info("session event: %s. reason: %s\n", xio_session_event_str(event_data->event), xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: server_data->connection = event_data->conn; break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); server_data->connection = NULL; break; case XIO_SESSION_TEARDOWN_EVENT: server_data->session = NULL; xio_session_destroy(session); if (atomic_read(&module_state) & 0x80) xio_context_stop_loop(server_data->ctx); /* exit */ break; default: break; }; return 0; }
static void xio_module_down(void *data) { struct test_params *params; struct xio_session *session; struct xio_connection *connection; params = (struct test_params *)data; if (!params->session) goto stop_loop_now; if (!params->connection) goto destroy_session; connection = params->connection; params->connection = NULL; xio_disconnect(connection); return; destroy_session: /* in multi thread version on need to user reference count */ session = params->session; params->session = NULL; xio_session_destroy(session); stop_loop_now: /* No session -> no XIO_SESSION_TEARDOWN_EVENT */ xio_context_stop_loop(params->ctx); /* exit */ }
static void xio_module_down(void *data) { struct xio_session *tmp_session; struct xio_connection *tmp_connection; if (!g_session) goto stop_loop_now; if (!g_connection) goto destroy_session; tmp_connection = g_connection; g_connection = NULL; xio_disconnect(tmp_connection); return; destroy_session: /* in multi thread version on need to user reference count */ tmp_session = g_session; g_session = NULL; xio_session_destroy(tmp_session); stop_loop_now: /* No session -> no XIO_SESSION_TEARDOWN_EVENT */ xio_context_stop_loop(ctx); /* exit */ }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { pr_info("session event: %s. reason: %s\n", xio_session_event_str(event_data->event), xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_CONNECTION_ESTABLISHED_EVENT: on_connection_established(event_data->conn); break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); break; case XIO_SESSION_REJECT_EVENT: case XIO_SESSION_TEARDOWN_EVENT: g_session = NULL; xio_session_destroy(session); xio_context_stop_loop(ctx); /* exit */ if (pool) { msg_pool_free(pool); pool = NULL; } break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { switch (event_data->event) { case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct xio_connection_attr conn_attr; struct test_params *test_params = cb_user_context; pr_info("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: /* assign connection private data */ conn_attr.user_context = cb_user_context; xio_modify_connection(event_data->conn, &conn_attr, XIO_CONNECTION_ATTR_USER_CTX); if (!test_params->connection) test_params->connection = event_data->conn; break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: if (event_data->reason != XIO_E_SESSION_REJECTED) { pr_info("last recv:%llu\n", test_params->nrecv); test_params->connection = NULL; } xio_connection_destroy(event_data->conn); test_params->connection = NULL; break; case XIO_SESSION_TEARDOWN_EVENT: test_params->session = NULL; xio_session_destroy(session); if (event_data->reason != XIO_E_SESSION_REJECTED) { if (atomic_read(&module_state) & 0x80) xio_context_stop_loop( test_params->ctx); /* exit */ } break; default: break; }; return 0; }
static int server_on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct client_info *ci; struct xio_connection_attr attr; struct server_data *server_data = (struct server_data *)cb_user_context; sd_debug("session event: %s. session:%p, connection:%p, reason: %s", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: server_data->nr_conn++; memset(&attr, 0, sizeof(attr)); ci = xio_create_client(session, event_data->conn); list_add_tail(&ci->conn.list, &server_data->conn_list); attr.user_context = ci; xio_modify_connection(event_data->conn, &attr, XIO_CONNECTION_ATTR_USER_CTX); xio_context_stop_loop(xio_get_main_ctx()); break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: server_data->nr_conn--; sd_assert(0 <= server_data->nr_conn); xio_connection_destroy(event_data->conn); xio_context_stop_loop(xio_get_main_ctx()); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); xio_context_stop_loop(xio_get_main_ctx()); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct server_data *sdata; struct thread_data *tdata; int i; sdata = (struct server_data *)cb_user_context; tdata = (event_data->conn_user_context == sdata) ? NULL : (struct thread_data *)event_data->conn_user_context; printf("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: if (tdata) tdata->connection = event_data->conn; break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); if (tdata) tdata->connection = NULL; break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); for (i = 0; i < sdata->tdata_nr; i++) { process_request(&sdata->tdata[i], NULL); xio_context_stop_loop(sdata->tdata[i].ctx); } xio_context_stop_loop((struct xio_context *)sdata->ctx); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct xio_connection_params cparams; struct test_params *test_params = cb_user_context; printf("session event: %s. session:%p, connection:%p, reason: %s\n", xio_session_event_str(event_data->event), session, event_data->conn, xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_NEW_CONNECTION_EVENT: /* assign connection private data */ cparams.user_context = cb_user_context; xio_set_connection_params(event_data->conn, &cparams); break; case XIO_SESSION_REJECT_EVENT: xio_disconnect(event_data->conn); break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: printf("last sent:%"PRIu64", last comp:%"PRIu64", " \ "delta:%"PRIu64"\n", test_params->nsent, test_params->ncomp, test_params->nsent-test_params->ncomp); xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); xio_context_stop_loop(test_params->ctx, 0); break; default: break; }; return 0; }
static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct client_data *client_data = (struct client_data *)cb_user_context; switch (event_data->event) { case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); xio_context_stop_loop(client_data->ctx); /* exit */ break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); xio_context_stop_loop(client_data->ctx); /* exit */ break; default: sd_debug("other event: %d", event_data->event); break; }; return 0; }
/*---------------------------------------------------------------------------*/ static int on_session_event(struct xio_session *session, struct xio_session_event_data *event_data, void *cb_user_context) { struct session_data *session_data = cb_user_context; printf("session event: %s. reason: %s\n", xio_session_event_str(event_data->event), xio_strerror(event_data->reason)); switch (event_data->event) { case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: xio_connection_destroy(event_data->conn); break; case XIO_SESSION_TEARDOWN_EVENT: xio_session_destroy(session); event_base_loopbreak(session_data->evbase); break; default: break; }; return 0; }
/*---------------------------------------------------------------------------*/ int run_client_test(struct perf_parameters *user_param) { struct session_data sess_data; struct perf_comm *comm; struct thread_data *tdata; char url[256]; int i = 0; int max_cpus; pthread_t statistics_thread_id; struct perf_command command; int size_log2; int max_size_log2 = 24; /* client session attributes */ struct xio_session_attr attr = { &ses_ops, NULL, 0 }; xio_init(); g_mhz = get_cpu_mhz(0); max_cpus = sysconf(_SC_NPROCESSORS_ONLN); threads_iter = 1; size_log2 = 0; tdata = calloc(user_param->threads_num, sizeof(*tdata)); if (tdata == NULL) { fprintf(fd, "malloc failed\n"); return -1; } comm = create_comm_struct(user_param); if (establish_connection(comm)) { fprintf(stderr, "failed to establish connection\n"); free(tdata); destroy_comm_struct(comm); return -1; } if (user_param->output_file) { fd = fopen(user_param->output_file, "w"); if (fd == NULL) { fprintf(fd, "file open failed. %s\n", user_param->output_file); free(sess_data.tdata); destroy_comm_struct(comm); return -1; } fprintf(fd, "size, threads, tps, bw[Mbps], lat[usec]\n"); fflush(fd); } printf("%s", RESULT_FMT); printf("%s", RESULT_LINE); while (threads_iter <= user_param->threads_num) { data_len = (uint64_t)1 << size_log2; memset(&sess_data, 0, sizeof(sess_data)); memset(tdata, 0, user_param->threads_num*sizeof(*tdata)); sess_data.tdata = tdata; command.test_param.machine_type = user_param->machine_type; command.test_param.test_type = user_param->test_type; command.test_param.verb = user_param->verb; command.test_param.data_len = data_len; command.command = GetTestParams; ctx_write_data(comm, &command, sizeof(command)); sprintf(url, "rdma://%s:%d", user_param->server_addr, user_param->server_port); sess_data.session = xio_session_create(XIO_SESSION_CLIENT, &attr, url, 0, 0, &sess_data); if (sess_data.session == NULL) { int error = xio_errno(); fprintf(stderr, "session creation failed. reason %d - (%s)\n", error, xio_strerror(error)); goto cleanup; } pthread_create(&statistics_thread_id, NULL, statistics_thread_cb, &sess_data); /* spawn threads to handle connection */ for (i = 0; i < threads_iter; i++) { sess_data.tdata[i].affinity = ((user_param->cpu + i) % max_cpus); sess_data.tdata[i].cid = i; sess_data.tdata[i].sdata = &sess_data; sess_data.tdata[i].user_param = user_param; sess_data.tdata[i].data_len = data_len; /* all threads are working on the same session */ sess_data.tdata[i].session = sess_data.session; pthread_create(&sess_data.tdata[i].thread_id, NULL, worker_thread, &sess_data.tdata[i]); } pthread_join(statistics_thread_id, NULL); /* join the threads */ for (i = 0; i < threads_iter; i++) pthread_join(sess_data.tdata[i].thread_id, NULL); /* close the session */ xio_session_destroy(sess_data.session); if (sess_data.abort) { fprintf(stderr, "program aborted\n"); goto cleanup; } /* send result to server */ command.results.bytes = data_len; command.results.threads = threads_iter; command.results.tps = sess_data.tps; command.results.avg_bw = sess_data.avg_bw; command.results.avg_lat = sess_data.avg_lat_us; command.results.min_lat = sess_data.min_lat_us; command.results.max_lat = sess_data.max_lat_us; command.command = GetTestResults; /* sync point */ ctx_write_data(comm, &command, sizeof(command)); printf(REPORT_FMT, data_len, threads_iter, sess_data.tps, sess_data.avg_bw, sess_data.avg_lat_us, sess_data.min_lat_us, sess_data.max_lat_us); if (fd) fprintf(fd, "%lu, %d, %lu, %.2lf, %.2lf\n", data_len, threads_iter, sess_data.tps, sess_data.avg_bw, sess_data.avg_lat_us); fflush(fd); /* sync point */ ctx_read_data(comm, NULL, 0, NULL); if (++size_log2 < max_size_log2) continue; threads_iter++; size_log2 = 0; } printf("%s", RESULT_LINE); cleanup: if (fd) fclose(fd); ctx_hand_shake(comm); ctx_close_connection(comm); destroy_comm_struct(comm); free(tdata); xio_shutdown(); return 0; }
/* first message after new connection are going trough the server */ static int xio_on_new_message(struct xio_server *server, struct xio_nexus *nexus, int event, union xio_nexus_event_data *event_data) { struct xio_session *session = NULL; struct xio_connection *connection = NULL; struct xio_connection *connection1 = NULL; struct xio_task *task; uint32_t tlv_type; struct xio_session_params params; int locked = 0; if (!server || !nexus || !event_data || !event_data->msg.task) { ERROR_LOG("server [new session]: failed " \ "invalid parameter\n"); return -1; } task = event_data->msg.task; params.type = XIO_SESSION_SERVER; params.initial_sn = 0; params.ses_ops = &server->ops; params.uri = server->uri; params.private_data = NULL; params.private_data_len = 0; params.user_context = server->cb_private_data; /* read the first message type */ tlv_type = xio_read_tlv_type(&event_data->msg.task->mbuf); if (tlv_type == XIO_SESSION_SETUP_REQ) { /* create new session */ session = xio_session_create(¶ms); if (!session) { ERROR_LOG("server [new session]: failed " \ " allocating session failed\n"); return -1; } DEBUG_LOG("server [new session]: server:%p, " \ "session:%p, nexus:%p ,session_id:%d\n", server, session, nexus, session->session_id); /* get transport class routines */ session->validators_cls = xio_nexus_get_validators_cls(nexus); connection = xio_session_alloc_connection(session, server->ctx, 0, server->cb_private_data); if (!connection) { ERROR_LOG("server failed to allocate new connection\n"); goto cleanup; } connection1 = xio_session_assign_nexus(session, nexus); if (!connection1) { ERROR_LOG("server failed to assign new connection\n"); goto cleanup1; } connection = connection1; xio_idr_add_uobj(usr_idr, session, "xio_session"); xio_idr_add_uobj(usr_idr, connection, "xio_connection"); xio_connection_set_state(connection, XIO_CONNECTION_STATE_ONLINE); xio_connection_keepalive_start(connection); task->session = session; task->connection = connection; } else if (tlv_type == XIO_CONNECTION_HELLO_REQ) { struct xio_session *session1; /* find the old session without lock */ session = xio_find_session(event_data->msg.task); if (!session) { ERROR_LOG("server [new connection]: failed " \ "session not found. server:%p\n", server); xio_nexus_close(nexus, NULL); return -1; } /* lock it and retry find */ mutex_lock(&session->lock); /* session before destruction - try to lock before continue */ session1 = xio_find_session(event_data->msg.task); if (!session1) { ERROR_LOG("server [new connection]: failed " \ "session not found. server:%p\n", server); xio_nexus_close(nexus, NULL); mutex_unlock(&session->lock); return -1; } locked = 1; task->session = session; DEBUG_LOG("server [new connection]: server:%p, " \ "session:%p, nexus:%p, session_id:%d\n", server, session, nexus, session->session_id); connection = xio_session_alloc_connection( task->session, server->ctx, 0, server->cb_private_data); if (!connection) { ERROR_LOG("server failed to allocate new connection\n"); goto cleanup; } connection1 = xio_session_assign_nexus(task->session, nexus); if (!connection1) { ERROR_LOG("server failed to assign new connection\n"); goto cleanup1; } connection = connection1; /* copy the server attributes to the connection */ xio_connection_set_ops(connection, &server->ops); task->connection = connection; /* This in a multiple-portal situation */ session->state = XIO_SESSION_STATE_ONLINE; xio_connection_set_state(connection, XIO_CONNECTION_STATE_ONLINE); xio_connection_keepalive_start(connection); xio_idr_add_uobj(usr_idr, connection, "xio_connection"); } else { ERROR_LOG("server unexpected message\n"); return -1; } /* route the message to the session */ if (session) xio_nexus_notify_observer(nexus, &session->observer, event, event_data); if (locked) mutex_unlock(&session->lock); return 0; cleanup1: if (connection) xio_session_free_connection(connection); cleanup: if (session) xio_session_destroy(session); return -1; }