int SRLockClient::start_client () {
SRClientContext ctx;
char temp_char;
ctx.ib_port = SERVER_IB_PORT;
srand (generate_random_seed()); // initialize random seed
TEST_NZ (establish_tcp_connection(SERVER_ADDR.c_str(), SERVER_TCP_PORT, &(ctx.sockfd)));
DEBUG_COUT("[Comm] Client connected to LM on sock " << ctx.sockfd);
TEST_NZ (ctx.create_context());
DEBUG_COUT("[Info] Context Created " << ctx.sockfd);
TEST_NZ (RDMACommon::connect_qp (&(ctx.qp), ctx.ib_port, ctx.port_attr.lid, ctx.sockfd));
DEBUG_COUT("[Conn] QP connected!");
start_operation(ctx);
// Sync so server will know that client is done mucking with its memory
DEBUG_COUT("[Info] Client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
int IPCohort::start_server () {
IPCohortContext ctx;
char temp_char;
TEST_NZ (establish_tcp_connection(TRX_MANAGER_ADDR.c_str(), TRX_MANAGER_TCP_PORT, &ctx.sockfd));
TEST_NZ (ctx.create_context());
srand (time(NULL)); // initialize random seed
start_benchmark(ctx);
/* Sync so server will know that client is done mucking with its memory */
DEBUG_COUT("[Info] Cohort client is done, and is ready to destroy its resources!");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
TEST_NZ(ctx.destroy_context());
}
int Coordinator::start () {
CoordinatorContext ctx[SERVER_CNT];
struct sockaddr_in returned_addr;
socklen_t len = sizeof(returned_addr);
char temp_char;
srand (generate_random_seed()); // initialize random seed
// Call socket(), bind() and listen()
TEST_NZ (server_socket_setup(&server_sockfd, SERVER_CNT));
// accept connections from Cohorts
std::cout << "[Info] Waiting for " << SERVER_CNT << " cohort(s) on port " << TRX_MANAGER_TCP_PORT << std::endl;
for (int s = 0; s < SERVER_CNT; s++) {
ctx[s].sockfd = accept (server_sockfd, (struct sockaddr *) &returned_addr, &len);
if (ctx[s].sockfd < 0){
std::cerr << "ERROR on accept() for RM #" << s << std::endl;
return -1;
}
std::cout << "[Conn] Received Cohort #" << s << " on sock " << ctx[s].sockfd << std::endl;
// create all resources
ctx[s].ib_port = TRX_MANAGER_IB_PORT;
TEST_NZ (ctx[s].create_context());
DEBUG_COUT("[Info] Context for cohort " << s << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[s].qp), ctx[s].ib_port, ctx[s].port_attr.lid, ctx[s].sockfd));
DEBUG_COUT("[Conn] QPed to cohort " << s);
}
std::cout << "[Info] Established connection to all " << SERVER_CNT << " resource-manager(s)." << std::endl;
start_benchmark(ctx);
DEBUG_COUT("[Info] Coordinator is done, and is ready to destroy its resources!");
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
DEBUG_COUT("[Conn] Notified cohort " << i << " it's done");
TEST_NZ ( ctx[i].destroy_context());
}
}
void* DQClientCentricServer::handle_client(void *param) {
DQServerContext *ctx = (DQServerContext *) param;
char temp_char;
DEBUG_COUT("[in handle client]");
// TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int)));
TEST_NZ (sock_sync_data (ctx->sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Synced with client]");
/*
int iteration = 0;
while (iteration < OPERATIONS_CNT) {
TEST_NZ (RDMACommon::poll_completion(ctx->cq));
DEBUG_COUT("[Recv] request from client");
TEST_NZ (RDMACommon::post_RECEIVE(ctx->qp, ctx->recv_data_mr, (uintptr_t)&ctx->recv_data_msg, sizeof(int))); // for
DEBUG_COUT("[Info] receive posted to the queue");
if (iteration % 1000 == 0) {
TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), true));
DEBUG_COUT("[Sent] response to client");
TEST_NZ (RDMACommon::poll_completion(ctx->cq)); // for SEND
DEBUG_COUT("[Info] completion received");
}
else {
TEST_NZ (RDMACommon::post_SEND(ctx->qp, ctx->send_data_mr, (uintptr_t)ctx->send_data_msg, 10 * sizeof(char), false));
DEBUG_COUT("[Sent] response to client (without completion)");
}
iteration++;
}
*/
DEBUG_COUT("[Sent] buffer info to client");
}
int RDMAServer::start_server (int server_num) {
tcp_port = TCP_PORT[server_num];
ib_port = IB_PORT[server_num];
RDMAServerContext ctx[CLIENTS_CNT];
char temp_char;
TEST_NZ(initialize_data_structures());
std::cout << "[Info] Server " << server_num << " is waiting for " << CLIENTS_CNT
<< " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl;
server_sockfd = -1;
struct sockaddr_in serv_addr, cli_addr;
socklen_t clilen = sizeof(cli_addr);
pthread_t master_threads[CLIENTS_CNT];
// Open Socket
server_sockfd = socket (AF_INET, SOCK_STREAM, 0);
if (server_sockfd < 0) {
std::cerr << "Error opening socket" << std::endl;
return -1;
}
// Bind
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(tcp_port);
TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)));
// listen
TEST_NZ(listen (server_sockfd, CLIENTS_CNT));
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
// pthread_create(&master_threads[i], NULL, RDMAServer::handle_client, &client_socks[i]);
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
// prepare server buffer with read message
memcpy(&(ctx[i].send_msg.mr_items), ctx[i].mr_items, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_orders), ctx[i].mr_orders, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_order_line),ctx[i].mr_order_line, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_cc_xacts), ctx[i].mr_cc_xacts, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_timestamp), ctx[i].mr_timestamp, sizeof(struct ibv_mr));
memcpy(&(ctx[i].send_msg.mr_lock_items),ctx[i].mr_lock_items, sizeof(struct ibv_mr));
}
for (int i = 0; i < CLIENTS_CNT; i++){
// send memory locations using SEND
TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_mr, (uintptr_t)&ctx[i].send_msg, sizeof(struct MemoryKeys), true));
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq));
DEBUG_COUT("[Sent] buffer info to client " << i);
}
/*
Server waits for the client to muck with its memory
*/
for (int i = 0; i < CLIENTS_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); /* just send a dummy char back and forth */
DEBUG_COUT("[Conn] Client " << i << " notified it's finished");
TEST_NZ (ctx[i].destroy_context());
std::cout << "[Info] Destroying client " << i << " resources" << std::endl;
}
std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl;
}
int RDMASock::sock_sync_ready(struct sock_t *sock)
{
char cm_buf = 'a';
return sock_sync_data(sock, sizeof(cm_buf), &cm_buf, &cm_buf);
}
int IPCohort::start_benchmark(IPCohortContext &ctx) {
char temp_char;
unsigned long long cpu_checkpoint_start, cpu_checkpoint_finish;
int iteration = 0;
struct rusage usage;
struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage;
DEBUG_COUT("[in handle client]");
TEST_NZ (sock_sync_data (ctx.sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Synced with client]");
// For CPU usage in terms of time
getrusage(RUSAGE_SELF, &usage);
start_kernel_usage = usage.ru_stime;
start_user_usage = usage.ru_utime;
// For CPU usage in terms of clocks (ticks)
cpu_checkpoint_start = rdtsc();
while (iteration < OPERATIONS_CNT) {
//TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive
TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int)));
DEBUG_COUT("[READ] --READY-- request from coordinator");
//TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), false));
TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int)));
DEBUG_COUT("[WRIT] --YES-- response to coordinator (without completion)");
//TEST_NZ (RDMACommon::poll_completion(ctx.cq)); // for Receive
TEST_NZ (sock_read(ctx.sockfd, (char *)&(ctx.recv_data_msg), sizeof(int)));
DEBUG_COUT("[READ] --COMMIT-- request from coordinator");
//TEST_NZ (RDMACommon::post_SEND(ctx.qp, ctx.send_data_mr, (uintptr_t)&ctx.send_data_msg, sizeof(int), true));
TEST_NZ (sock_write(ctx.sockfd, (char *)&(ctx.send_data_msg), sizeof(int)));
DEBUG_COUT("[WRIT] --DONE-- response to coordinator");
iteration++;
}
cpu_checkpoint_finish = rdtsc();
getrusage(RUSAGE_SELF, &usage);
end_user_usage = usage.ru_utime;
end_kernel_usage = usage.ru_stime;
double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec );
double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec );
DEBUG_COUT("[Info] Cohort Client Done");
unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT;
std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl;
return 0;
}
static int connect_qp(struct resources *res)
{
struct cm_con_data_t local_con_data;
struct cm_con_data_t remote_con_data;
struct cm_con_data_t tmp_con_data;
int rc;
/* modify the QP to init */
rc = modify_qp_to_init(res->qp);
if (rc) {
fprintf(stderr, "change QP state to INIT failed\n");
return rc;
}
/* let the client post RR to be prepared for incoming messages */
if (config.server_name) {
rc = post_receive(res);
if (rc) {
fprintf(stderr, "failed to post RR\n");
return rc;
}
}
/* exchange using TCP sockets info required to connect QPs */
local_con_data.qp_num = htonl(res->qp->qp_num);
local_con_data.lid = htons(res->port_attr.lid);
fprintf(stdout, "\nLocal LID = 0x%x\n", res->port_attr.lid);
if (sock_sync_data(res->sock, !config.server_name, sizeof(struct cm_con_data_t), &local_con_data, &tmp_con_data) < 0) {
fprintf(stderr, "failed to exchange connection data between sides\n");
return 1;
}
remote_con_data.qp_num = ntohl(tmp_con_data.qp_num);
remote_con_data.lid = ntohs(tmp_con_data.lid);
fprintf(stdout, "Remote QP number = 0x%x\n", remote_con_data.qp_num);
fprintf(stdout, "Remote LID = 0x%x\n", remote_con_data.lid);
/* modify the QP to RTR */
rc = modify_qp_to_rtr(res->qp, remote_con_data.qp_num, remote_con_data.lid);
if (rc) {
fprintf(stderr, "failed to modify QP state from RESET to RTS\n");
return rc;
}
/* only the daemon post SR, so only he should be in RTS
(the client can be moved to RTS as well)
*/
if (config.server_name)
fprintf(stdout, "QP state was change to RTR\n");
else {
rc = modify_qp_to_rts(res->qp);
if (rc) {
fprintf(stderr, "failed to modify QP state from RESET to RTS\n");
return rc;
}
fprintf(stdout, "QP state was change to RTS\n");
}
/* sync to make sure that both sides are in states that they can connect to prevent packet loose */
if (sock_sync_ready(res->sock, !config.server_name)) {
fprintf(stderr, "sync after QPs are were moved to RTS\n");
return 1;
}
return 0;
}
int DQClientCentricServer::start_server () {
tcp_port = SERVER_TCP_PORT;
ib_port = SERVER_IB_PORT;
DQServerContext ctx[CLIENTS_CNT];
struct sockaddr_in serv_addr, cli_addr;
socklen_t clilen = sizeof(cli_addr);
char temp_char;
std::string clients_addresses; // separated by the delimiter '|'
std::string clients_tcp_port; // separated by the delimiter '|'
TEST_NZ(initialize_data_structures());
std::cout << "[Info] Server is waiting for " << CLIENTS_CNT
<< " client(s) on tcp port: " << tcp_port << ", ib port: " << ib_port << std::endl;
// Open Socket
server_sockfd = socket (AF_INET, SOCK_STREAM, 0);
if (server_sockfd < 0) {
std::cerr << "Error opening socket" << std::endl;
return -1;
}
// Bind
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(tcp_port);
TEST_NZ(bind(server_sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)));
// listen
TEST_NZ(listen (server_sockfd, CLIENTS_CNT));
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
ctx[i].client_ip = "";
ctx[i].client_ip += std::string(inet_ntoa (cli_addr.sin_addr));
clients_addresses.append(ctx[i].client_ip);
clients_tcp_port.append(std::to_string(8000 + i));
// if the client is not the last client, add the delimiter
if (i != CLIENTS_CNT - 1){
clients_addresses.append("|");
clients_tcp_port.append("|");
}
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " (" << ctx[i].client_ip << ") created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
}
for (int i = 0; i < CLIENTS_CNT; i++){
// send memory locations using SEND
memcpy(&(ctx[i].send_message_msg.peer_mr), ctx[i].locks_mr, sizeof(struct ibv_mr));
ctx[i].send_message_msg.client_id = static_cast<uint32_t>(i + 1);
strcpy(ctx[i].send_message_msg.clients_addr, clients_addresses.c_str());
strcpy(ctx[i].send_message_msg.clients_tcp_port, clients_tcp_port.c_str());
DEBUG_COUT("[Sent] Cheeck: " << ctx[i].send_message_msg.clients_addr);
TEST_NZ (RDMACommon::post_SEND (ctx[i].qp, ctx[i].send_message_mr, (uintptr_t)&ctx[i].send_message_msg, sizeof (struct MemoryKeys), true));
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq));
DEBUG_COUT("[Sent] buffer info to client " << i);
}
// Server waits for the client to muck with its memory
/*************** THIS IS FOR SEND
// accept connections
for (int i = 0; i < CLIENTS_CNT; i++){
initialize_context(ctx[i]);
ctx[i].sockfd = accept (server_sockfd, (struct sockaddr *) &cli_addr, &clilen);
if (ctx[i].sockfd < 0){
std::cerr << "ERROR on accept" << std::endl;
return -1;
}
std::cout << "[Conn] Received client #" << i << " on socket " << ctx[i].sockfd << std::endl;
// create all resources
TEST_NZ (ctx[i].create_context());
DEBUG_COUT("[Info] Context for client " << i << " created");
// connect the QPs
TEST_NZ (RDMACommon::connect_qp (&(ctx[i].qp), ctx[i].ib_port, ctx[i].port_attr.lid, ctx[i].sockfd));
DEBUG_COUT("[Conn] QPed to client " << i);
pthread_create(&master_threads[i], NULL, BenchmarkServer::handle_client, &ctx[i]);
}
std::cout << "[Info] Established connection to all " << CLIENTS_CNT << " client(s)." << std::endl;
//wait for handlers to finish
for (int i = 0; i < CLIENTS_CNT; i++) {
pthread_join(master_threads[i], NULL);
}
*/
for (int i = 0; i < CLIENTS_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
DEBUG_COUT("[Conn] Client " << i << " notified it's finished");
TEST_NZ (ctx[i].destroy_context());
std::cout << "[Info] Destroying client " << i << " resources" << std::endl;
}
std::cout << "[Info] Server's ready to gracefully get destroyed" << std::endl;
}
static void* rdma_thread(void *ptr)
{
int i, j, rc;
struct rdma_resource_t *rdma_resource;
struct user_param_t *user_param;
struct thread_context_t *t_ctx;
struct rdma_req_t rdma_req;
double lat;
t_ctx = (struct thread_context_t*)ptr;
rdma_resource = t_ctx->rdma_resource;
user_param = &(rdma_resource->user_param);
t_ctx->thread_id = pthread_self();
t_ctx->num_of_iter = user_param->num_of_iter;
if (create_rdma_buf_pool(t_ctx)) {
ERROR("Failed to create MR pool.\n");
return NULL;
}
{
uint32_t qp_type;
if (user_param->server_ip != NULL) {
qp_type = htonl(user_param->qp_type);
}
sock_c2d(&(t_ctx->sock), sizeof(qp_type), &qp_type);
if (user_param->server_ip == NULL) {
user_param->qp_type = ntohl(qp_type);
}
t_ctx->qp_type = user_param->qp_type; /// redesign
}
if (create_qp(t_ctx)) {
ERROR("Failed to create QP.\n");
return NULL;
}
{
struct thread_sync_info_t {
uint32_t qp_num;
uint32_t direction;
uint32_t opcode;
uint32_t qkey;
uint32_t psn;
uint32_t num_of_iter;
uint16_t lid;
} ATTR_PACKED;
struct thread_sync_info_t local_info;
struct thread_sync_info_t remote_info;
local_info.lid = htons(rdma_resource->port_attr.lid);
local_info.qp_num = htonl(t_ctx->qp->qp_num);
local_info.direction = htonl(user_param->direction);
local_info.opcode = htonl(user_param->opcode); /// enum ibv_wr_opcode
local_info.qkey = htonl(0);
local_info.psn = htonl(0);
local_info.num_of_iter = htonl(t_ctx->num_of_iter);
rc = sock_sync_data(&(t_ctx->sock), sizeof(local_info), &local_info, &remote_info);
if (rc) {
ERROR("failed to sync data.\n");
return NULL;
}
t_ctx->remote_lid = ntohs(remote_info.lid);
t_ctx->remote_qpn = ntohl(remote_info.qp_num);
t_ctx->remote_qkey = ntohl(remote_info.qkey);
t_ctx->remote_psn = ntohl(remote_info.psn);
if (user_param->server_ip == NULL) {
user_param->direction = ntohl(remote_info.direction);
user_param->opcode = ntohl(remote_info.opcode);
t_ctx->num_of_iter = ntohl(remote_info.num_of_iter);
if (user_param->direction == 0 || user_param->direction == 1) {
t_ctx->is_requestor = 0;
} else if (user_param->direction == 2) {
t_ctx->is_requestor = 1;
}
} else {
if (user_param->direction == 0 || user_param->direction == 1) {
t_ctx->is_requestor = 1;
} else if (user_param->direction == 2) {
t_ctx->is_requestor = 0;
}
}
}
t_ctx->t_a = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t));
if (t_ctx->t_a == NULL) {
ERROR("Failed to allocate memory.\n");
return NULL;
}
t_ctx->t_b = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t));
if (t_ctx->t_b == NULL) {
free(t_ctx->t_a);
ERROR("Failed to allocate memory.\n");
return NULL;
}
t_ctx->t_c = (cycles_t*)malloc(t_ctx->num_of_iter * sizeof(cycles_t));
if (t_ctx->t_c == NULL) {
free(t_ctx->t_b);
free(t_ctx->t_a);
ERROR("Failed to allocate memory.\n");
return NULL;
}
for (i = 0; i < LAT_LEVEL; i++) {
t_ctx->lat[i] = 0;
}
if (connect_qp(t_ctx)) {
ERROR("Failed to connect QP.\n");
return NULL;
}
for(i = 0; i < user_param->num_of_oust; i++) {
rdma_req.rdma_buf = get_rdma_buf(t_ctx);
rdma_req.num_of_oust = 1;
rdma_req.data_size = DEF_BUF_SIZE;
rc = post_receive(t_ctx, &rdma_req);
if (rc) {
ERROR("Failed to post_receive, i:%d.\n", i);
return NULL;
}
}
sock_sync_ready(&t_ctx->sock);
for (i = 0; i < t_ctx->num_of_iter; i++) {
t_ctx->t_a[i] = get_cycles();
DEBUG("do_rdma_transaction, t_ctx->num_of_iter=%d, i=%d.\n", t_ctx->num_of_iter, i);
rc = do_rdma_transaction(t_ctx, i);
if (rc) {
ERROR("Failed to do_rdma_transaction, i:%d.\n", i);
return NULL;
}
t_ctx->t_c[i] = get_cycles();
if (user_param->direction == 0 || (!t_ctx->is_requestor)) {
rdma_req.rdma_buf = get_rdma_buf(t_ctx);
if (rdma_req.rdma_buf == NULL) {
ERROR("Failed to get RDMA buffer.\n");
return NULL; /// Memory Leak and remove hung RX buffers
}
rdma_req.num_of_oust = 1;
post_receive(t_ctx, &rdma_req);
}
if (user_param->interval) {
usleep(user_param->interval);
}
}
/// Memory leak, release the hung RX rdma_buf;
destroy_qp(t_ctx);
t_ctx->min_lat = 0x7fffffff;
t_ctx->max_lat = 0;
for (i = 0; i < t_ctx->num_of_iter; i++) {
lat = (t_ctx->t_c[i] - t_ctx->t_a[i]) / rdma_resource->freq_mhz;
if (lat < t_ctx->min_lat) {
t_ctx->min_lat = lat;
t_ctx->min_lat_iter_num = i;
}
if (lat > t_ctx->max_lat) {
t_ctx->max_lat = lat;
t_ctx->max_lat_iter_num = i;
}
for (j = 0; j < LAT_LEVEL; j++) {
if (j < 7) {
if (lat < (1 + j)) {
t_ctx->lat[j]++;
break;
}
} else {
if (lat < (1 << (j - 4))) {
t_ctx->lat[j]++;
break;
}
}
}
if (j == LAT_LEVEL) {
t_ctx->lat[LAT_LEVEL - 1]++;
}
}
free(t_ctx->t_a);
free(t_ctx->t_b);
free(t_ctx->t_c);
if (!user_param->server_ip) {
/// sock_close_multi(&(t_ctx->sock), sock_bind); // how to close sock_fd.
free(t_ctx); /// Need to improve.
}
INFO("RDMA testing thread successfully exited.\n");
return NULL;
}
int Coordinator::start_benchmark(CoordinatorContext *ctx) {
int signaledPosts = 0;
struct timespec firstRequestTime, lastRequestTime; // for calculating TPMS
struct timespec beforeSending, afterSending; // for calculating TPMS
struct timespec before_read_ts, after_read_ts, after_fetch_info, after_commit_ts, after_lock, after_decrement, after_unlock;
struct rusage usage;
struct timeval start_user_usage, start_kernel_usage, end_user_usage, end_kernel_usage;
char temp_char;
unsigned long long cpu_checkpoint_start, cpu_checkpoint_2, cpu_checkpoint_3, cpu_checkpoint_finish ;
bool signalled_flag = false;
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (sock_sync_data (ctx[i].sockfd, 1, "W", &temp_char)); // just send a dummy char back and forth
}
DEBUG_COUT ("[Info] Benchmark now gets started");
clock_gettime(CLOCK_REALTIME, &firstRequestTime); // Fire the timer
getrusage(RUSAGE_SELF, &usage);
start_kernel_usage = usage.ru_stime;
start_user_usage = usage.ru_utime;
int iteration = 0;
cpu_checkpoint_start = rdtsc();
while (iteration < OPERATIONS_CNT) {
DEBUG_COUT("iteration " << iteration);
// Sent: Ready??
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int)));
DEBUG_COUT("[Info] Receive posted");
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false));
DEBUG_COUT("[Sent] --READY-- request sent to cohort " << i);
}
// Received: Ready!
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV
DEBUG_COUT("[Recv] --YES-- response received from cohort " << i);
}
// Sent: Commit
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::post_RECEIVE(ctx[i].qp, ctx[i].recv_data_mr, (uintptr_t)&ctx[i].recv_data_msg, sizeof(int)));
DEBUG_COUT("[Info] Receive posted");
if (iteration % 500 == 0) {
signalled_flag = true;
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), true));
DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort " << i);
}
else{
signalled_flag = false;
TEST_NZ (RDMACommon::post_SEND(ctx[i].qp, ctx[i].send_data_mr, (uintptr_t)&ctx[i].send_data_msg, sizeof(int), false));
DEBUG_COUT("[Sent] --COMMIT-- request sent to cohort (unsignalled) " << i);
}
}
// collect completion events for SEND
if (signalled_flag){
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for SENT
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for SENT
}
}
// Received: Done!
for (int i = 0; i < SERVER_CNT; i++) {
TEST_NZ (RDMACommon::poll_completion(ctx[i].cq)); // for RECV
//TEST_NZ (RDMACommon::event_based_poll_completion(ctx[i].comp_channel, ctx[i].cq)); // for RECV
DEBUG_COUT("[Recv] --DONE-- response received from cohort " << i);
}
iteration++;
}
cpu_checkpoint_finish = rdtsc();
getrusage(RUSAGE_SELF, &usage);
end_user_usage = usage.ru_utime;
end_kernel_usage = usage.ru_stime;
clock_gettime(CLOCK_REALTIME, &lastRequestTime); // Fire the timer
double user_cpu_microtime = ( end_user_usage.tv_sec - start_user_usage.tv_sec ) * 1E6 + ( end_user_usage.tv_usec - start_user_usage.tv_usec );
double kernel_cpu_microtime = ( end_kernel_usage.tv_sec - start_kernel_usage.tv_sec ) * 1E6 + ( end_kernel_usage.tv_usec - start_kernel_usage.tv_usec );
double micro_elapsed_time = ( ( lastRequestTime.tv_sec - firstRequestTime.tv_sec ) * 1E9 + ( lastRequestTime.tv_nsec - firstRequestTime.tv_nsec ) ) / 1000;
double latency_in_micro = (double)(micro_elapsed_time / OPERATIONS_CNT);
//double latency_in_micro = (double)(cumulative_latency / signaledPosts) / 1000;
double mega_byte_per_sec = ((sizeof(int) * OPERATIONS_CNT / 1E6 ) / (micro_elapsed_time / 1E6) );
double operations_per_sec = OPERATIONS_CNT / (micro_elapsed_time / 1E6);
double cpu_utilization = (user_cpu_microtime + kernel_cpu_microtime) / micro_elapsed_time;
unsigned long long average_cpu_clocks = (cpu_checkpoint_finish - cpu_checkpoint_start) / OPERATIONS_CNT;
std::cout << "[Stat] Avg latency(u sec): " << latency_in_micro << std::endl;
std::cout << "[Stat] MegaByte per Sec: " << mega_byte_per_sec << std::endl;
std::cout << "[Stat] Operations per Sec: " << operations_per_sec << std::endl;
std::cout << "[Stat] CPU utilization: " << cpu_utilization << std::endl;
std::cout << "[Stat] USER CPU utilization: " << user_cpu_microtime / micro_elapsed_time << std::endl;
std::cout << "[Stat] KERNEL CPU utilization: " << kernel_cpu_microtime / micro_elapsed_time << std::endl;
std::cout << "[Stat] AVG USER CPU elapsed time (u sec): " << user_cpu_microtime / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] AVG KERNEL CPU elapsed time (u sec): " << kernel_cpu_microtime / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] AVG total CPU elapsed time (u sec): " << (user_cpu_microtime + kernel_cpu_microtime) / OPERATIONS_CNT << std::endl;
std::cout << "[Stat] Average CPU clocks: " << average_cpu_clocks << std::endl;
std::cout << latency_in_micro << '\t' << mega_byte_per_sec << '\t' << operations_per_sec << '\t' << cpu_utilization << std::endl;
return 0;
}
