static int do_client(struct options *opts, bmi_context_id *context) { int ret = 0; int i = 0; PVFS_BMI_addr_t peer_addr; void *recv_buffer = NULL; void *send_buffer = NULL; bmi_op_id_t op_id[2]; bmi_error_code_t error_code; int outcount = 0; bmi_size_t actual_size; struct msg *tx_msg = NULL; int bytes = MIN_BYTES; int max_bytes = MAX_BYTES; int warmup = 1; int iterations = 0; int msg_len = 0; int run = 0; #ifdef WIN32 LARGE_INTEGER start; LARGE_INTEGER end; #else struct timeval start; struct timeval end; #endif double *val = NULL; double lat = 0.0; double min = 99999.9; double max = 0.0; double avg = 0.0; int offset; #ifdef HAVE_LIBZ unsigned long crc=0, rcrc=0; #endif /* get a bmi_addr for the server */ ret = BMI_addr_lookup(&peer_addr, opts->hostid); if (ret < 0) { errno = -ret; perror("BMI_addr_lookup"); return (-1); } if (opts->test == UNEXPECTED) { ret = BMI_get_info(peer_addr, BMI_GET_UNEXP_SIZE, (void *)&max_bytes); if (ret < 0) { fprintf(stderr, "BMI_get_info() returned %d\n", ret); return ret; } } else { int maxsize = 0; ret = BMI_get_info(peer_addr, BMI_CHECK_MAXSIZE, (void *)&maxsize); if (ret < 0) { fprintf(stderr, "BMI_get_info() returned %d\n", ret); return ret; } if (maxsize < max_bytes) max_bytes = maxsize; } msg_len = sizeof(struct msg); /* create send buffer */ send_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_SEND); if (!send_buffer) { fprintf(stderr, "BMI_memalloc failed.\n"); return (-1); } if(opts->crc) { for(i = 0; i < max_bytes; ++i) { ((char *)send_buffer)[i] = i; } } else { memset(send_buffer, 0, max_bytes); } tx_msg = (struct msg *) send_buffer; tx_msg->test = htonl(opts->test); /* create a buffer to recv into */ recv_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_RECV); if (!recv_buffer) { fprintf(stderr, "BMI_memalloc failed.\n"); return (-1); } /* post the test parameters */ ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr, tx_msg, msg_len, BMI_PRE_ALLOC, 0, NULL, *context, NULL); if (ret < 0) { fprintf(stderr, "BMI_post_sendunexpected failure.\n"); return (-1); } else if (ret == 0) { do { ret = BMI_test(op_id[SEND], &outcount, &error_code, &actual_size, NULL, 10, *context); } while (ret == 0 && outcount == 0); if (ret < 0 || error_code != 0) { fprintf(stderr, "data send failed.\n"); return (-1); } if (actual_size != msg_len) { fprintf(stderr, "Expected %d but received %llu\n", msg_len, llu(actual_size)); return (-1); } } /* post a recv for the ack */ ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer, msg_len, &actual_size, BMI_PRE_ALLOC, 0, NULL, *context, NULL); if (ret < 0) { fprintf(stderr, "BMI_post_recv_failure.\n"); return (-1); } else if (ret == 0) { do { ret = BMI_test(op_id[RECV], &outcount, &error_code, &actual_size, NULL, 10, *context); } while (ret == 0 && outcount == 0); if (ret < 0 || error_code != 0) { fprintf(stderr, "data recv failed.\n"); return (-1); } if (actual_size != msg_len) { fprintf(stderr, "Expected %d but received %llu\n", msg_len, llu(actual_size)); return (-1); } } val = calloc(ITERATIONS, sizeof(double)); if (val == NULL) { fprintf(stderr, "calloc() for val failed\n"); return -1; } /* make sure server has posted first recv */ #ifdef WIN32 Sleep(1000); #else sleep(1); #endif fprintf(stdout, " Bytes usecs MB/s StdDev Min Max\n"); /* start iterations */ while (bytes <= max_bytes) { iterations = bytes_to_iterations(bytes); for (i=0; i < iterations; i++) { #ifdef WIN32 offset = rand() % (max_bytes - bytes - 1); #else offset = random() % (max_bytes - bytes - 1); #endif #ifdef WIN32 QueryPerformanceCounter(&start); #else gettimeofday(&start, NULL); #endif #ifdef HAVE_LIBZ if(opts->crc) { crc = adler32(0L, Z_NULL, 0); crc = adler32(crc, ((unsigned char *)send_buffer + (unsigned int)offset), bytes); } #endif /* post the recv for the pong */ ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer, bytes, &actual_size, BMI_PRE_ALLOC, i, NULL, *context, NULL); if (ret < 0) { fprintf(stderr, "BMI_post_recv_failure.\n"); return (-1); } /* send the ping */ if (opts->test == EXPECTED) { ret = BMI_post_send(&(op_id[SEND]), peer_addr, ((char *)send_buffer) + offset, bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL); } else { ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr, ((char*)send_buffer) + offset, bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL); } if (ret < 0) { fprintf(stderr, "BMI_post_sendunexpected failure.\n"); return (-1); } else if (ret == 0) { do { ret = BMI_test(op_id[SEND], &outcount, &error_code, &actual_size, NULL, 10, *context); } while (ret == 0 && outcount == 0); if (ret < 0 || error_code != 0) { fprintf(stderr, "send ping failed.\n"); return (-1); } if (actual_size != bytes) { fprintf(stderr, "Expected %d but received %llu\n", bytes, llu(actual_size)); return (-1); } } /* complete the receive for the pong */ do { ret = BMI_test(op_id[RECV], &outcount, &error_code, &actual_size, NULL, 10, *context); } while (ret == 0 && outcount == 0); if (ret < 0 || error_code != 0) { fprintf(stderr, "data recv failed.\n"); return (-1); } if (actual_size != bytes) { fprintf(stderr, "Expected %d but received %llu\n", bytes, llu(actual_size)); return (-1); } #ifdef HAVE_LIBZ if(opts->crc && opts->test == EXPECTED) { rcrc = adler32(0L, Z_NULL, 0); rcrc = adler32(rcrc, recv_buffer, bytes); if(rcrc != crc) { fprintf(stderr, "CRC Mismatch! " "Sent %llu but received %llu\n", llu(crc), llu(rcrc)); } } #endif #ifdef WIN32 QueryPerformanceCounter(&end); #else gettimeofday(&end, NULL); #endif if (!warmup) { #ifdef WIN32 val[i] = ((double) end.QuadPart - (double) start.QuadPart) / (double) freq.QuadPart; #else val[i] = (double) end.tv_sec + (double) end.tv_usec * 0.000001; val[i] -= (double) start.tv_sec + (double) start.tv_usec * 0.000001; #endif lat += val[i]; } } if (!warmup) { double stdev = 0.0; lat = lat / (double) iterations * 1000000.0 / 2.0; min = 999999.9; max = 0.0; avg = 0.0; /* convert seconds to MB/s */ for (i=0; i < iterations; i++) { val[i] = (double) bytes * 2 / val[i] / 1000000.0; avg += val[i]; if (val[i] < min) min = val[i]; if (val[i] > max) max = val[i]; } avg /= iterations; if (iterations > 1) { for (i=0; i < iterations; i++) { double diff = val[i] - avg; stdev += diff * diff; } stdev = sqrt(stdev / (iterations - 1)); } fprintf(stdout, "%10d %12.3f %12.3f +- %9.3f %12.3f %12.3f\n", bytes, lat, avg, stdev, min, max); lat = 0.0; bytes *= 2; run++; } else warmup = 0; } /* free up the message buffers */ BMI_memfree(peer_addr, recv_buffer, max_bytes, BMI_RECV); BMI_memfree(peer_addr, send_buffer, max_bytes, BMI_SEND); return ret; }
int main( int argc, char **argv) { int ret = -1; int outcount = 0; void *mybuffer; PVFS_BMI_addr_t server_addr; bmi_op_id_t op; bmi_error_code_t error_code; flow_descriptor *flow_d = NULL; int i = 0; bmi_size_t actual_size; double time1, time2; PINT_Request *req; bmi_context_id context; /*************************************************************/ /* initialization stuff */ /* set debugging level */ gossip_enable_stderr(); gossip_set_debug_mask( 0, (GOSSIP_FLOW_PROTO_DEBUG | GOSSIP_BMI_DEBUG_TCP)); /* Dist init */ PINT_dist_initialize(NULL); /* start up BMI */ ret = BMI_initialize("bmi_tcp", NULL, 0); if (ret < 0) { fprintf(stderr, "BMI init failure.\n"); return (-1); } ret = BMI_open_context(&context); if (ret < 0) { fprintf(stderr, "BMI_open_context() failure.\n"); return (-1); } /* initialize the flow interface */ ret = PINT_flow_initialize("flowproto_multiqueue", 0); if (ret < 0) { fprintf(stderr, "flow init failure.\n"); return (-1); } /* send some random crap to the other side to start up communication */ ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3335"); if (ret < 0) { fprintf(stderr, "BMI lookup failure.\n"); return (-1); } ret = BMI_post_sendunexpected(&op, server_addr, &mybuffer, 1, BMI_EXT_ALLOC, 0, NULL, context, NULL); if (ret < 0) { fprintf(stderr, "BMI_post_sendunexpected failure.\n"); return (-1); } if (ret == 0) { /* turning this into a blocking call for testing :) */ /* check for completion of request */ do { ret = BMI_test(op, &outcount, &error_code, &actual_size, NULL, 10, context); } while (ret == 0 && outcount == 0); if (ret < 0 || error_code != 0) { fprintf(stderr, "Request send failed.\n"); if (ret < 0) { errno = -ret; perror("BMI_test"); } return (-1); } } /******************************************************/ /* setup request/dist stuff */ /* request description */ /* just want one contiguous region */ ret = PVFS_Request_contiguous(TEST_SIZE, PVFS_BYTE, &req); if (ret < 0) { fprintf(stderr, "PVFS_Request_contiguous() failure.\n"); return (-1); } /******************************************************/ /* setup communicaton stuff */ /* memory buffer to xfer */ mybuffer = (void *) malloc(TEST_SIZE); if (!mybuffer) { fprintf(stderr, "mem.\n"); return (-1); } /* mark it so that we can check correctness */ for (i = 0; i < (TEST_SIZE / (sizeof(int))); i++) { ((int *) mybuffer)[i] = i; } /* create a flow descriptor */ flow_d = PINT_flow_alloc(); if (!flow_d) { fprintf(stderr, "mem.\n"); return (-1); } /* file data */ flow_d->file_data.fsize = TEST_SIZE; flow_d->file_data.server_nr = 0; flow_d->file_data.server_ct = 1; flow_d->file_data.extend_flag = 0; flow_d->file_data.dist = PINT_dist_create("basic_dist"); if (!flow_d->file_data.dist) { fprintf(stderr, "Error: failed to create dist.\n"); return (-1); } ret = PINT_dist_lookup(flow_d->file_data.dist); if (ret != 0) { fprintf(stderr, "Error: failed to lookup dist.\n"); return (-1); } flow_d->file_req = req; flow_d->tag = 0; flow_d->user_ptr = NULL; flow_d->aggregate_size = TEST_SIZE; /* fill in flow details */ flow_d->src.endpoint_id = MEM_ENDPOINT; flow_d->src.u.mem.buffer = mybuffer; flow_d->dest.endpoint_id = BMI_ENDPOINT; flow_d->dest.u.bmi.address = server_addr; /*************************************************** * test memory to bmi (analogous to client side write) */ time1 = Wtime(); ret = block_on_flow(flow_d); if (ret < 0) { return (-1); } time2 = Wtime(); /*******************************************************/ /* final cleanup and output */ #if 0 printf("Client bw (send): %f MB/sec\n", ((TEST_SIZE) / ((time2 - time1) * 1000000.0))); #endif PINT_flow_free(flow_d); /* shut down flow interface */ ret = PINT_flow_finalize(); if (ret < 0) { fprintf(stderr, "flow finalize failure.\n"); return (-1); } /* shut down BMI */ BMI_close_context(context); BMI_finalize(); free(mybuffer); gossip_disable(); return (0); }