/** * @brief Test the timer. * * An easy way to test the timer is to time the sleep function * for a fixed amount of time. If the timer results a time that * matches the sleep time within a reasonable error, the timer works. * * @return 0 on success, 1 on error. */ int trios_timer_test() { int rc = 0; static const uint64_t sleep_us = 1000000; /* 1 second */ /* TODO: what should be the acceptable error (1/2 sec?) */ static const uint64_t error_ms = 500; uint64_t error_ns = error_ms * 1000000; uint64_t error_us = error_ms * 1000; double error_sec = (double)error_ms/1000.0; uint64_t start_ns=0, t_ns=0; uint64_t start_us, t_us; uint64_t start_ms, t_ms; double start_sec, t_sec; log_debug(timer_debug_level, "this is a timer test. expect a 1 second delay. starting..."); /* run to initialize timer */ start_ns = trios_get_time_ns(); if (usleep(sleep_us)) { log_error(timer_debug_level, "Failed calling usleep()"); return 1; } /* run to get measurements */ start_ns = trios_get_time_ns(); start_us = trios_get_time_us(); start_ms = trios_get_time_ms(); start_sec = trios_get_time(); /* sleep one seconds */ usleep(sleep_us); t_ns = trios_get_time_ns() - start_ns; t_us = trios_get_time_us() - start_us; t_ms = trios_get_time_ms() - start_ms; t_sec = trios_get_time() - start_sec; printf("slept for %llu seconds:\n", sleep_us); printf("\tns = %llu\n", t_ns); printf("\tus = %llu\n", t_us); printf("\tms = %llu\n", t_ms); printf("\tsec = %f\n", t_sec); /* Make sure our values have a reasonable error */ if (labs(t_ns - (sleep_us * 1e3)) > error_ns) { uint64_t actual_err = labs(t_ns - (sleep_us *1e3)); log_error(timer_debug_level, "trios_timer failed ns timer test: err = %llu ns" " > acceptable err = %llu", actual_err, error_ns); rc = 1; } if (labs(t_us - sleep_us) > error_us) { uint64_t actual_err = labs(t_us - sleep_us); log_error(timer_debug_level, "trios_timer failed usec timer test: err = %llu usec" " > acceptable err = %llu", actual_err, error_us); rc |= 1; } if (labs(t_ms - (sleep_us / 1e3)) > error_ms) { uint64_t actual_err = labs(t_ms - (sleep_us / 1e3)); log_error(timer_debug_level, "trios_timer failed ns timer test: err = %llu ms" " > acceptable err = %llu", actual_err, error_ms); rc |= 1; } if (fabs(t_sec - ((double)sleep_us / 1e6)) > error_sec) { double actual_err = fabs(t_sec - ((double)sleep_us/1e6)); log_error(timer_debug_level, "trios_timer failed sec timer test: err = %g sec " " > acceptable err = %g", actual_err, error_sec); rc |= 1; } log_debug(timer_debug_level, "timer test complete"); return rc; }
void client(void) { NNTI_result_t rc=NNTI_OK; NNTI_status_t rdma_status; NNTI_status_t send_status; NNTI_status_t client_ack_status; void *packed=NULL; int32_t packed_size=0; double op_timer; // Teuchos::oblackholestream blackhole; // std::ostream &out = ( rank == 1 ? std::cout : blackhole ); std::ostream &out = std::cout; NNTI_connect(&trans_hdl, url, 5000, &server_hdl); char *send_buf=(char *)malloc(NNTI_REQUEST_BUFFER_SIZE); memset(send_buf, 0, NNTI_REQUEST_BUFFER_SIZE); NNTI_register_memory(&trans_hdl, send_buf, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, NULL, &send_mr); char *client_ack_buf=(char *)malloc(NNTI_REQUEST_BUFFER_SIZE); memset(client_ack_buf, 0, NNTI_REQUEST_BUFFER_SIZE); NNTI_register_memory(&trans_hdl, client_ack_buf, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_RECV_DST, NULL, &client_ack_mr); char *get_dst_buf=(char *)malloc(get_size); memset(get_dst_buf, 0, get_size); NNTI_register_memory(&trans_hdl, get_dst_buf, get_size, 1, NNTI_GET_DST, NULL, &get_dst_mr); char *put_src_buf=(char *)malloc(put_size); memset(put_src_buf, 0, put_size); NNTI_register_memory(&trans_hdl, put_src_buf, put_size, 1, NNTI_PUT_SRC, NULL, &put_src_mr); /* * Phase 1 - exchange buffer handles */ buffer_pack(&client_ack_mr, &packed, &packed_size, (xdrproc_t)&xdr_NNTI_buffer_t); if (packed_size > NNTI_REQUEST_BUFFER_SIZE) { log_error(nntiperf_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); MPI_Abort(MPI_COMM_WORLD, -10); } // send the server the recv_mr so it can send back it's ack_mr memcpy(send_buf, packed, packed_size); buffer_pack_free(packed, packed_size, (xdrproc_t)&xdr_NNTI_buffer_t); rc=NNTI_send(&server_hdl, &send_mr, NULL); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_send() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&send_mr, NNTI_SEND_SRC, 5000, &send_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } // wait for the server to send back it's recv_mr rc=NNTI_wait(&client_ack_mr, NNTI_RECV_DST, -1, &client_ack_status); char *ptr=(char*)client_ack_status.start+client_ack_status.offset; memcpy(&packed_size, ptr, sizeof(packed_size)); ptr += sizeof(packed_size); memcpy(packed, ptr, packed_size); ptr += packed_size; buffer_unpack(packed, packed_size, &server_ack_mr, (xdrproc_t)&xdr_NNTI_buffer_t); memcpy(&packed_size, ptr, sizeof(packed_size)); ptr += sizeof(packed_size); memcpy(packed, ptr, packed_size); ptr += packed_size; buffer_unpack(packed, packed_size, &get_src_mr, (xdrproc_t)&xdr_NNTI_buffer_t); memcpy(&packed_size, ptr, sizeof(packed_size)); ptr += sizeof(packed_size); memcpy(packed, ptr, packed_size); ptr += packed_size; buffer_unpack(packed, packed_size, &put_dst_mr, (xdrproc_t)&xdr_NNTI_buffer_t); // fprint_NNTI_buffer(logger_get_file(), "server_ack_mr", // "received server ack hdl", &server_ack_mr); // fprint_NNTI_buffer(logger_get_file(), "get_src_mr", // "received get src hdl", &get_src_mr); // fprint_NNTI_buffer(logger_get_file(), "put_dst_mr", // "received put dst hdl", &put_dst_mr); MPI_Barrier(MPI_COMM_WORLD); /* * Phase 2 - test sync request performance */ op_timer=trios_get_time(); for (int i=0;i<num_sends;i++) { rc=NNTI_send(&server_hdl, &send_mr, NULL); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_send() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&send_mr, NNTI_SEND_SRC, 1000, &send_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_sends > 0) { out << " sync requests per second == " << num_sends/op_timer << std::endl; } MPI_Barrier(MPI_COMM_WORLD); /* * Phase 3 - test async request performance */ op_timer=trios_get_time(); for (int i=0;i<num_sends;i++) { rc=NNTI_send(&server_hdl, &send_mr, NULL); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_send() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } for (int i=0;i<num_sends;i++) { rc=NNTI_wait(&send_mr, NNTI_SEND_SRC, 1000, &send_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_sends > 0) { out << "async requests per second == " << num_sends/op_timer << std::endl; } MPI_Barrier(MPI_COMM_WORLD); /* * Phase 4 - test sync get performance */ // warm up the pipes for (int i=0;i<num_gets;i++) { rc=NNTI_get(&get_src_mr, client_rank*get_size, get_size, &get_dst_mr, 0); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_get() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&get_dst_mr, NNTI_GET_DST, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time(); for (int i=0;i<num_gets;i++) { rc=NNTI_get(&get_src_mr, client_rank*get_size, get_size, &get_dst_mr, 0); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_get() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&get_dst_mr, NNTI_GET_DST, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_gets > 0) { out << " sync get (" << get_size << " byte transfer) == " << (double)(num_gets*get_size)/one_mb/op_timer << " MBps" << std::endl; } MPI_Barrier(MPI_COMM_WORLD); /* * Phase 5 - test async get performance */ // warm up the pipes for (int i=0;i<num_gets;i++) { rc=NNTI_get(&get_src_mr, client_rank*get_size, get_size, &get_dst_mr, 0); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_get() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } for (int i=0;i<num_gets;i++) { rc=NNTI_wait(&get_dst_mr, NNTI_GET_DST, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time(); for (int i=0;i<num_gets;i++) { rc=NNTI_get(&get_src_mr, client_rank*get_size, get_size, &get_dst_mr, 0); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_get() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } for (int i=0;i<num_gets;i++) { rc=NNTI_wait(&get_dst_mr, NNTI_GET_DST, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_gets > 0) { out << "async get (" << get_size << " byte transfer) == " << (double)(num_gets*get_size)/one_mb/op_timer << " MBps" << std::endl; } MPI_Barrier(MPI_COMM_WORLD); /* * Phase 6 - test sync put performance */ // warm up the pipes for (int i=0;i<num_puts;i++) { rc=NNTI_put(&put_src_mr, 0, put_size, &put_dst_mr, client_rank*put_size); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_put() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&put_src_mr, NNTI_PUT_SRC, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time(); for (int i=0;i<num_puts;i++) { rc=NNTI_put(&put_src_mr, 0, put_size, &put_dst_mr, client_rank*put_size); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_put() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } rc=NNTI_wait(&put_src_mr, NNTI_PUT_SRC, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_puts > 0) { out << " sync put (" << put_size << " byte transfer) == " << (double)(num_puts*put_size)/one_mb/op_timer << " MBps" << std::endl; } MPI_Barrier(MPI_COMM_WORLD); /* * Phase 7 - test async put performance */ // warm up the pipes for (int i=0;i<num_puts;i++) { rc=NNTI_put(&put_src_mr, 0, put_size, &put_dst_mr, client_rank*put_size); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_put() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } for (int i=0;i<num_puts;i++) { rc=NNTI_wait(&put_src_mr, NNTI_PUT_SRC, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time(); for (int i=0;i<num_puts;i++) { rc=NNTI_put(&put_src_mr, 0, put_size, &put_dst_mr, client_rank*put_size); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_put() returned an error: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } for (int i=0;i<num_puts;i++) { rc=NNTI_wait(&put_src_mr, NNTI_PUT_SRC, 1000, &rdma_status); if (rc != NNTI_OK) { log_error(nntiperf_debug_level, "NNTI_wait() did not return NNTI_OK: %d", rc); MPI_Abort(MPI_COMM_WORLD, rc); } } op_timer=trios_get_time()-op_timer; if (num_puts > 0) { out << "async put (" << put_size << " byte transfer) == " << (double)(num_puts*put_size)/one_mb/op_timer << " MBps" << std::endl; } MPI_Barrier(MPI_COMM_WORLD); NNTI_unregister_memory(&send_mr); free(send_buf); NNTI_unregister_memory(&client_ack_mr); free(client_ack_buf); NNTI_unregister_memory(&get_dst_mr); free(get_dst_buf); NNTI_unregister_memory(&put_src_mr); free(put_src_buf); return; }