int SRLockClient::start_client () { SRClientContext ctx; char temp_char; ctx.ib_port = SERVER_IB_PORT; srand (generate_random_seed()); // initialize random seed TEST_NZ (establish_tcp_connection(SERVER_ADDR.c_str(), SERVER_TCP_PORT, &(ctx.sockfd))); DEBUG_COUT("[Comm] Client connected to LM on sock " << ctx.sockfd); TEST_NZ (ctx.create_context()); DEBUG_COUT("[Info] Context Created " << ctx.sockfd); TEST_NZ (RDMACommon::connect_qp (&(ctx.qp), ctx.ib_port, ctx.port_attr.lid, ctx.sockfd)); DEBUG_COUT("[Conn] QP connected!"); start_operation(ctx); // Sync so server will know that client is done mucking with its memory DEBUG_COUT("[Info] Client is done, and is ready to destroy its resources!"); TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */ TEST_NZ(ctx.destroy_context()); }
int IPCohort::start_server () { IPCohortContext ctx; char temp_char; TEST_NZ (establish_tcp_connection(TRX_MANAGER_ADDR.c_str(), TRX_MANAGER_TCP_PORT, &ctx.sockfd)); TEST_NZ (ctx.create_context()); srand (time(NULL)); // initialize random seed start_benchmark(ctx); /* Sync so server will know that client is done mucking with its memory */ DEBUG_COUT("[Info] Cohort client is done, and is ready to destroy its resources!"); TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */ TEST_NZ(ctx.destroy_context()); }
int Coordinator::start () { CoordinatorContext ctx[SERVER_CNT]; struct sockaddr_in returned_addr; socklen_t len = sizeof(returned_addr); char temp_char; srand (generate_random_seed()); // initialize random seed // Call socket(), bind() and listen() TEST_NZ (server_socket_setup(&server_sockfd, SERVER_CNT)); // accept connections from Cohorts std::cout << "[Info] Waiting for " << SERVER_CNT << " cohort(s) on port " << TRX_MANAGER_TCP_PORT << std::endl; for (int s = 0; s < SERVER_CNT; s++) { ctx[s].sockfd = accept (server_sockfd, (struct sockaddr *) &returned_addr, &len); if (ctx[s].sockfd < 0){ std::cerr << "ERROR on accept() for RM #" << s << std::endl; return -1; } std::cout << "[Conn] Received Cohort #" << s << " on sock " << ctx[s].sockfd << std::endl; // create all resources ctx[s].ib_port = TRX_MANAGER_IB_PORT; TEST_NZ (ctx[s].create_context()); DEBUG_COUT("[Info] Context for cohort " << s << " created"); // connect the QPs TEST_NZ (RDMACommon::connect_qp (&(ctx[s].qp), ctx[s].ib_port, ctx[s].port_attr.lid, ctx[s].sockfd)); DEBUG_COUT("[Conn] QPed to cohort " << s); } std::cout << "[Info] Established connection to all " << SERVER_CNT << " resource-manager(s)." << std::endl; start_benchmark(ctx); DEBUG_COUT("[Info] Coordinator is done, and is ready to destroy its resources!"); for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */ DEBUG_COUT("[Conn] Notified cohort " << i << " it's done"); TEST_NZ ( ctx[i].destroy_context()); } }
void* DQClientCentricServer::handle_client(void *param) { DQServerContext *ctx = (DQServerContext *) param; char temp_char; DEBUG_COUT("[in handle client]"); // TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int))); TEST_NZ (sock_sync_data (ctx->sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth DEBUG_COUT("[Synced with client]"); /* int iteration = 0; while (iteration < OPERATIONS_CNT) { TEST_NZ (RDMACommon::poll_completion(ctx->cq)); DEBUG_COUT("[Recv] request from client"); TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int))); // for DEBUG_COUT("[Info] receive posted to the queue"); if (iteration % 1000 == 0) { TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), true)); DEBUG_COUT("[Sent] response to client"); TEST_NZ (RDMACommon::poll_completion(ctx->cq)); // for SEND DEBUG_COUT("[Info] completion received"); } else { TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), false)); DEBUG_COUT("[Sent] response to client (without completion)"); } iteration++; } */ DEBUG_COUT("[Sent] buffer info to client"); }
int RDMAServer::start_server (int server_num) { tcp_port = TCP_PORT[server_num]; ib_port = IB_PORT[server_num]; RDMAServerContext ctx[CLIENTS_CNT]; char temp_char; TEST_NZ(initialize_data_structures()); std::cout << "[Info] Server " << server_num << " is waiting for " << CLIENTS_CNT << " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl; server_sockfd = -1; struct sockaddr_in serv_addr, cli_addr; socklen_t clilen = sizeof(cli_addr); pthread_t master_threads[CLIENTS_CNT]; // Open Socket server_sockfd = socket (AF_INET, SOCK_STREAM, 0); if (server_sockfd < 0) { std::cerr << "Error opening socket" << std::endl; return -1; } // Bind memset(&serv_addr, 0, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; serv_addr.sin_addr.s_addr = INADDR_ANY; serv_addr.sin_port = htons(tcp_port); TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr))); // listen TEST_NZ(listen (server_sockfd, CLIENTS_CNT)); // accept connections for (int i = 0; i < CLIENTS_CNT; i++){ initialize_context(ctx[i]); ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen); if (ctx[i].sockfd < 0){ std::cerr << "ERROR on accept" << std::endl; return -1; } std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl; // pthread_create(&master_threads[i], NULL, RDMAServer::handle_client, &client_socks[i]); // create all resources TEST_NZ (ctx[i].create_context()); DEBUG_COUT("[Info] Context for client " << i << " created"); // connect the QPs TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd)); DEBUG_COUT("[Conn] QPed to client " << i); // prepare server buffer with read message memcpy(&(ctx[i].send_msg.mr_items), ctx[i].mr_items, sizeof(struct ibv_mr)); memcpy(&(ctx[i].send_msg.mr_orders), ctx[i].mr_orders, sizeof(struct ibv_mr)); memcpy(&(ctx[i].send_msg.mr_order_line),ctx[i].mr_order_line, sizeof(struct ibv_mr)); memcpy(&(ctx[i].send_msg.mr_cc_xacts), ctx[i].mr_cc_xacts, sizeof(struct ibv_mr)); memcpy(&(ctx[i].send_msg.mr_timestamp), ctx[i].mr_timestamp, sizeof(struct ibv_mr)); memcpy(&(ctx[i].send_msg.mr_lock_items),ctx[i].mr_lock_items, sizeof(struct ibv_mr)); } for (int i = 0; i < CLIENTS_CNT; i++){ // send memory locations using SEND TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_mr, (uintptr_t)&ctx[i].send_msg, sizeof(struct MemoryKeys), true)); TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); DEBUG_COUT("[Sent] buffer info to client " << i); } /* Server waits for the client to muck with its memory */ for (int i = 0; i < CLIENTS_CNT; i++) { TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */ DEBUG_COUT("[Conn] Client " << i << " notified it's finished"); TEST_NZ (ctx[i].destroy_context()); std::cout << "[Info] Destroying client " << i << " resources" << std::endl; } std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl; }
int RDMASock::sock_sync_ready(struct sock_t *sock) { char cm_buf = 'a'; return sock_sync_data(sock, sizeof(cm_buf), &cm_buf, &cm_buf); }
int IPCohort::start_benchmark(IPCohortContext &ctx) { char temp_char; unsigned long long cpu_checkpoint_start, cpu_checkpoint_finish; int iteration = 0; struct rusage usage; struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage; DEBUG_COUT("[in handle client]"); TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth DEBUG_COUT("[Synced with client]"); // For CPU usage in terms of time getrusage(RUSAGE_SELF, &usage); start_kernel_usage = usage.ru_stime; start_user_usage = usage.ru_utime; // For CPU usage in terms of clocks (ticks) cpu_checkpoint_start = rdtsc(); while (iteration < OPERATIONS_CNT) { //TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int))); DEBUG_COUT("[READ] --READY-- request from coordinator"); //TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), false)); TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int))); DEBUG_COUT("[WRIT] --YES-- response to coordinator (without completion)"); //TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int))); DEBUG_COUT("[READ] --COMMIT-- request from coordinator"); //TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), true)); TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int))); DEBUG_COUT("[WRIT] --DONE-- response to coordinator"); iteration++; } cpu_checkpoint_finish = rdtsc(); getrusage(RUSAGE_SELF, &usage); end_user_usage = usage.ru_utime; end_kernel_usage = usage.ru_stime; double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec ); double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec ); DEBUG_COUT("[Info] Cohort Client Done"); unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT; std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl; std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl; return 0; }
static int connect_qp(struct resources *res) { struct cm_con_data_t local_con_data; struct cm_con_data_t remote_con_data; struct cm_con_data_t tmp_con_data; int rc; /* modify the QP to init */ rc = modify_qp_to_init(res->qp); if (rc) { fprintf(stderr, "change QP state to INIT failed\n"); return rc; } /* let the client post RR to be prepared for incoming messages */ if (config.server_name) { rc = post_receive(res); if (rc) { fprintf(stderr, "failed to post RR\n"); return rc; } } /* exchange using TCP sockets info required to connect QPs */ local_con_data.qp_num = htonl(res->qp->qp_num); local_con_data.lid = htons(res->port_attr.lid); fprintf(stdout, "\nLocal LID = 0x%x\n", res->port_attr.lid); if (sock_sync_data(res->sock, !config.server_name, sizeof(struct cm_con_data_t), &local_con_data, &tmp_con_data) < 0) { fprintf(stderr, "failed to exchange connection data between sides\n"); return 1; } remote_con_data.qp_num = ntohl(tmp_con_data.qp_num); remote_con_data.lid = ntohs(tmp_con_data.lid); fprintf(stdout, "Remote QP number = 0x%x\n", remote_con_data.qp_num); fprintf(stdout, "Remote LID = 0x%x\n", remote_con_data.lid); /* modify the QP to RTR */ rc = modify_qp_to_rtr(res->qp, remote_con_data.qp_num, remote_con_data.lid); if (rc) { fprintf(stderr, "failed to modify QP state from RESET to RTS\n"); return rc; } /* only the daemon post SR, so only he should be in RTS (the client can be moved to RTS as well) */ if (config.server_name) fprintf(stdout, "QP state was change to RTR\n"); else { rc = modify_qp_to_rts(res->qp); if (rc) { fprintf(stderr, "failed to modify QP state from RESET to RTS\n"); return rc; } fprintf(stdout, "QP state was change to RTS\n"); } /* sync to make sure that both sides are in states that they can connect to prevent packet loose */ if (sock_sync_ready(res->sock, !config.server_name)) { fprintf(stderr, "sync after QPs are were moved to RTS\n"); return 1; } return 0; }
int DQClientCentricServer::start_server () { tcp_port = SERVER_TCP_PORT; ib_port = SERVER_IB_PORT; DQServerContext ctx[CLIENTS_CNT]; struct sockaddr_in serv_addr, cli_addr; socklen_t clilen = sizeof(cli_addr); char temp_char; std::string clients_addresses; // separated by the delimiter '|' std::string clients_tcp_port; // separated by the delimiter '|' TEST_NZ(initialize_data_structures()); std::cout << "[Info] Server is waiting for " << CLIENTS_CNT << " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl; // Open Socket server_sockfd = socket (AF_INET, SOCK_STREAM, 0); if (server_sockfd < 0) { std::cerr << "Error opening socket" << std::endl; return -1; } // Bind memset(&serv_addr, 0, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; serv_addr.sin_addr.s_addr = INADDR_ANY; serv_addr.sin_port = htons(tcp_port); TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr))); // listen TEST_NZ(listen (server_sockfd, CLIENTS_CNT)); // accept connections for (int i = 0; i < CLIENTS_CNT; i++){ initialize_context(ctx[i]); ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen); if (ctx[i].sockfd < 0){ std::cerr << "ERROR on accept" << std::endl; return -1; } std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl; ctx[i].client_ip = ""; ctx[i].client_ip += std::string(inet_ntoa (cli_addr.sin_addr)); clients_addresses.append(ctx[i].client_ip); clients_tcp_port.append(std::to_string(8000 + i)); // if the client is not the last client, add the delimiter if (i != CLIENTS_CNT - 1){ clients_addresses.append("|"); clients_tcp_port.append("|"); } // create all resources TEST_NZ (ctx[i].create_context()); DEBUG_COUT("[Info] Context for client " << i << " (" << ctx[i].client_ip << ") created"); // connect the QPs TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd)); DEBUG_COUT("[Conn] QPed to client " << i); } for (int i = 0; i < CLIENTS_CNT; i++){ // send memory locations using SEND memcpy(&(ctx[i].send_message_msg.peer_mr), ctx[i].locks_mr, sizeof(struct ibv_mr)); ctx[i].send_message_msg.client_id = static_cast<uint32_t>(i + 1); strcpy(ctx[i].send_message_msg.clients_addr, clients_addresses.c_str()); strcpy(ctx[i].send_message_msg.clients_tcp_port, clients_tcp_port.c_str()); DEBUG_COUT("[Sent] Cheeck: " << ctx[i].send_message_msg.clients_addr); TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_message_mr, (uintptr_t)&ctx[i].send_message_msg, sizeof (struct MemoryKeys), true)); TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); DEBUG_COUT("[Sent] buffer info to client " << i); } // Server waits for the client to muck with its memory /*************** THIS IS FOR SEND // accept connections for (int i = 0; i < CLIENTS_CNT; i++){ initialize_context(ctx[i]); ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen); if (ctx[i].sockfd < 0){ std::cerr << "ERROR on accept" << std::endl; return -1; } std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl; // create all resources TEST_NZ (ctx[i].create_context()); DEBUG_COUT("[Info] Context for client " << i << " created"); // connect the QPs TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd)); DEBUG_COUT("[Conn] QPed to client " << i); pthread_create(&master_threads[i], NULL, BenchmarkServer::handle_client, &ctx[i]); } std::cout << "[Info] Established connection to all " << CLIENTS_CNT << " client(s)." << std::endl; //wait for handlers to finish for (int i = 0; i < CLIENTS_CNT; i++) { pthread_join(master_threads[i], NULL); } */ for (int i = 0; i < CLIENTS_CNT; i++) { TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth DEBUG_COUT("[Conn] Client " << i << " notified it's finished"); TEST_NZ (ctx[i].destroy_context()); std::cout << "[Info] Destroying client " << i << " resources" << std::endl; } std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl; }
static void* rdma_thread(void *ptr) { int i, j, rc; struct rdma_resource_t *rdma_resource; struct user_param_t *user_param; struct thread_context_t *t_ctx; struct rdma_req_t rdma_req; double lat; t_ctx = (struct thread_context_t*)ptr; rdma_resource = t_ctx->rdma_resource; user_param = &(rdma_resource->user_param); t_ctx->thread_id = pthread_self(); t_ctx->num_of_iter = user_param->num_of_iter; if (create_rdma_buf_pool(t_ctx)) { ERROR("Failed to create MR pool.\n"); return NULL; } { uint32_t qp_type; if (user_param->server_ip != NULL) { qp_type = htonl(user_param->qp_type); } sock_c2d(&(t_ctx->sock), sizeof(qp_type), &qp_type); if (user_param->server_ip == NULL) { user_param->qp_type = ntohl(qp_type); } t_ctx->qp_type = user_param->qp_type; /// redesign } if (create_qp(t_ctx)) { ERROR("Failed to create QP.\n"); return NULL; } { struct thread_sync_info_t { uint32_t qp_num; uint32_t direction; uint32_t opcode; uint32_t qkey; uint32_t psn; uint32_t num_of_iter; uint16_t lid; } ATTR_PACKED; struct thread_sync_info_t local_info; struct thread_sync_info_t remote_info; local_info.lid = htons(rdma_resource->port_attr.lid); local_info.qp_num = htonl(t_ctx->qp->qp_num); local_info.direction = htonl(user_param->direction); local_info.opcode = htonl(user_param->opcode); /// enum ibv_wr_opcode local_info.qkey = htonl(0); local_info.psn = htonl(0); local_info.num_of_iter = htonl(t_ctx->num_of_iter); rc = sock_sync_data(&(t_ctx->sock), sizeof(local_info), &local_info, &remote_info); if (rc) { ERROR("failed to sync data.\n"); return NULL; } t_ctx->remote_lid = ntohs(remote_info.lid); t_ctx->remote_qpn = ntohl(remote_info.qp_num); t_ctx->remote_qkey = ntohl(remote_info.qkey); t_ctx->remote_psn = ntohl(remote_info.psn); if (user_param->server_ip == NULL) { user_param->direction = ntohl(remote_info.direction); user_param->opcode = ntohl(remote_info.opcode); t_ctx->num_of_iter = ntohl(remote_info.num_of_iter); if (user_param->direction == 0 || user_param->direction == 1) { t_ctx->is_requestor = 0; } else if (user_param->direction == 2) { t_ctx->is_requestor = 1; } } else { if (user_param->direction == 0 || user_param->direction == 1) { t_ctx->is_requestor = 1; } else if (user_param->direction == 2) { t_ctx->is_requestor = 0; } } } t_ctx->t_a = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t)); if (t_ctx->t_a == NULL) { ERROR("Failed to allocate memory.\n"); return NULL; } t_ctx->t_b = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t)); if (t_ctx->t_b == NULL) { free(t_ctx->t_a); ERROR("Failed to allocate memory.\n"); return NULL; } t_ctx->t_c = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t)); if (t_ctx->t_c == NULL) { free(t_ctx->t_b); free(t_ctx->t_a); ERROR("Failed to allocate memory.\n"); return NULL; } for (i = 0; i < LAT_LEVEL; i++) { t_ctx->lat[i] = 0; } if (connect_qp(t_ctx)) { ERROR("Failed to connect QP.\n"); return NULL; } for(i = 0; i < user_param->num_of_oust; i++) { rdma_req.rdma_buf = get_rdma_buf(t_ctx); rdma_req.num_of_oust = 1; rdma_req.data_size = DEF_BUF_SIZE; rc = post_receive(t_ctx, &rdma_req); if (rc) { ERROR("Failed to post_receive, i:%d.\n", i); return NULL; } } sock_sync_ready(&t_ctx->sock); for (i = 0; i < t_ctx->num_of_iter; i++) { t_ctx->t_a[i] = get_cycles(); DEBUG("do_rdma_transaction, t_ctx->num_of_iter=%d, i=%d.\n", t_ctx->num_of_iter, i); rc = do_rdma_transaction(t_ctx, i); if (rc) { ERROR("Failed to do_rdma_transaction, i:%d.\n", i); return NULL; } t_ctx->t_c[i] = get_cycles(); if (user_param->direction == 0 || (!t_ctx->is_requestor)) { rdma_req.rdma_buf = get_rdma_buf(t_ctx); if (rdma_req.rdma_buf == NULL) { ERROR("Failed to get RDMA buffer.\n"); return NULL; /// Memory Leak and remove hung RX buffers } rdma_req.num_of_oust = 1; post_receive(t_ctx, &rdma_req); } if (user_param->interval) { usleep(user_param->interval); } } /// Memory leak, release the hung RX rdma_buf; destroy_qp(t_ctx); t_ctx->min_lat = 0x7fffffff; t_ctx->max_lat = 0; for (i = 0; i < t_ctx->num_of_iter; i++) { lat = (t_ctx->t_c[i] - t_ctx->t_a[i]) / rdma_resource->freq_mhz; if (lat < t_ctx->min_lat) { t_ctx->min_lat = lat; t_ctx->min_lat_iter_num = i; } if (lat > t_ctx->max_lat) { t_ctx->max_lat = lat; t_ctx->max_lat_iter_num = i; } for (j = 0; j < LAT_LEVEL; j++) { if (j < 7) { if (lat < (1 + j)) { t_ctx->lat[j]++; break; } } else { if (lat < (1 << (j - 4))) { t_ctx->lat[j]++; break; } } } if (j == LAT_LEVEL) { t_ctx->lat[LAT_LEVEL - 1]++; } } free(t_ctx->t_a); free(t_ctx->t_b); free(t_ctx->t_c); if (!user_param->server_ip) { /// sock_close_multi(&(t_ctx->sock), sock_bind); // how to close sock_fd. free(t_ctx); /// Need to improve. } INFO("RDMA testing thread successfully exited.\n"); return NULL; }
int Coordinator::start_benchmark(CoordinatorContext *ctx) { int signaledPosts = 0; struct timespec firstRequestTime, lastRequestTime; // for calculating TPMS struct timespec beforeSending, afterSending; // for calculating TPMS struct timespec before_read_ts, after_read_ts, after_fetch_info, after_commit_ts, after_lock, after_decrement, after_unlock; struct rusage usage; struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage; char temp_char; unsigned long long cpu_checkpoint_start, cpu_checkpoint_2, cpu_checkpoint_3, cpu_checkpoint_finish ; bool signalled_flag = false; for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth } DEBUG_COUT ("[Info] Benchmark now gets started"); clock_gettime(CLOCK_REALTIME, &firstRequestTime); // Fire the timer getrusage(RUSAGE_SELF, &usage); start_kernel_usage = usage.ru_stime; start_user_usage = usage.ru_utime; int iteration = 0; cpu_checkpoint_start = rdtsc(); while (iteration < OPERATIONS_CNT) { DEBUG_COUT("iteration " << iteration); // Sent: Ready?? for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int))); DEBUG_COUT("[Info] Receive posted"); TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false)); DEBUG_COUT("[Sent] --READY-- request sent to cohort " << i); } // Received: Ready! for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV //TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV DEBUG_COUT("[Recv] --YES-- response received from cohort " << i); } // Sent: Commit for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int))); DEBUG_COUT("[Info] Receive posted"); if (iteration % 500 == 0) { signalled_flag = true; TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), true)); DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort " << i); } else{ signalled_flag = false; TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false)); DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort (unsignalled) " << i); } } // collect completion events for SEND if (signalled_flag){ for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for SENT //TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for SENT } } // Received: Done! for (int i = 0; i < SERVER_CNT; i++) { TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV //TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV DEBUG_COUT("[Recv] --DONE-- response received from cohort " << i); } iteration++; } cpu_checkpoint_finish = rdtsc(); getrusage(RUSAGE_SELF, &usage); end_user_usage = usage.ru_utime; end_kernel_usage = usage.ru_stime; clock_gettime(CLOCK_REALTIME, &lastRequestTime); // Fire the timer double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec ); double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec ); double micro_elapsed_time = ( ( lastRequestTime.tv_sec - firstRequestTime.tv_sec ) * 1E9 + ( lastRequestTime.tv_nsec - firstRequestTime.tv_nsec ) ) / 1000; double latency_in_micro = (double)(micro_elapsed_time / OPERATIONS_CNT); //double latency_in_micro = (double)(cumulative_latency / signaledPosts) / 1000; double mega_byte_per_sec = ((sizeof(int) * OPERATIONS_CNT / 1E6 ) / (micro_elapsed_time / 1E6) ); double operations_per_sec = OPERATIONS_CNT / (micro_elapsed_time / 1E6); double cpu_utilization = (user_cpu_microtime + kernel_cpu_microtime) / micro_elapsed_time; unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT; std::cout << "[Stat] Avg latency(u sec): " << latency_in_micro << std::endl; std::cout << "[Stat] MegaByte per Sec: " << mega_byte_per_sec << std::endl; std::cout << "[Stat] Operations per Sec: " << operations_per_sec << std::endl; std::cout << "[Stat] CPU utilization: " << cpu_utilization << std::endl; std::cout << "[Stat] USER CPU utilization: " << user_cpu_microtime / micro_elapsed_time << std::endl; std::cout << "[Stat] KERNEL CPU utilization: " << kernel_cpu_microtime / micro_elapsed_time << std::endl; std::cout << "[Stat] AVG USER CPU elapsed time (u sec): " << user_cpu_microtime / OPERATIONS_CNT << std::endl; std::cout << "[Stat] AVG KERNEL CPU elapsed time (u sec): " << kernel_cpu_microtime / OPERATIONS_CNT << std::endl; std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl; std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl; std::cout << latency_in_micro << '\t' << mega_byte_per_sec << '\t' << operations_per_sec << '\t' << cpu_utilization << std::endl; return 0; }