static int do_client(struct options *opts, bmi_context_id *context)
{
int ret = 0;
int i = 0;
PVFS_BMI_addr_t peer_addr;
void *recv_buffer = NULL;
void *send_buffer = NULL;
bmi_op_id_t op_id[2];
bmi_error_code_t error_code;
int outcount = 0;
bmi_size_t actual_size;
struct msg *tx_msg = NULL;
int bytes = MIN_BYTES;
int max_bytes = MAX_BYTES;
int warmup = 1;
int iterations = 0;
int msg_len = 0;
int run = 0;
#ifdef WIN32
LARGE_INTEGER start;
LARGE_INTEGER end;
#else
struct timeval start;
struct timeval end;
#endif
double *val = NULL;
double lat = 0.0;
double min = 99999.9;
double max = 0.0;
double avg = 0.0;
int offset;
#ifdef HAVE_LIBZ
unsigned long crc=0, rcrc=0;
#endif
/* get a bmi_addr for the server */
ret = BMI_addr_lookup(&peer_addr, opts->hostid);
if (ret < 0) {
errno = -ret;
perror("BMI_addr_lookup");
return (-1);
}
if (opts->test == UNEXPECTED) {
ret = BMI_get_info(peer_addr, BMI_GET_UNEXP_SIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
} else {
int maxsize = 0;
ret = BMI_get_info(peer_addr, BMI_CHECK_MAXSIZE,
(void *)&maxsize);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (maxsize < max_bytes) max_bytes = maxsize;
}
msg_len = sizeof(struct msg);
/* create send buffer */
send_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_SEND);
if (!send_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
if(opts->crc)
{
for(i = 0; i < max_bytes; ++i)
{
((char *)send_buffer)[i] = i;
}
}
else
{
memset(send_buffer, 0, max_bytes);
}
tx_msg = (struct msg *) send_buffer;
tx_msg->test = htonl(opts->test);
/* create a buffer to recv into */
recv_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_RECV);
if (!recv_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
/* post the test parameters */
ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr, tx_msg,
msg_len, BMI_PRE_ALLOC, 0, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data send failed.\n");
return (-1);
}
if (actual_size != msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
return (-1);
}
}
/* post a recv for the ack */
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
msg_len, &actual_size, BMI_PRE_ALLOC, 0, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
return (-1);
}
}
val = calloc(ITERATIONS, sizeof(double));
if (val == NULL) {
fprintf(stderr, "calloc() for val failed\n");
return -1;
}
/* make sure server has posted first recv */
#ifdef WIN32
Sleep(1000);
#else
sleep(1);
#endif
fprintf(stdout, " Bytes usecs MB/s StdDev Min Max\n");
/* start iterations */
while (bytes <= max_bytes) {
iterations = bytes_to_iterations(bytes);
for (i=0; i < iterations; i++) {
#ifdef WIN32
offset = rand() % (max_bytes - bytes - 1);
#else
offset = random() % (max_bytes - bytes - 1);
#endif
#ifdef WIN32
QueryPerformanceCounter(&start);
#else
gettimeofday(&start, NULL);
#endif
#ifdef HAVE_LIBZ
if(opts->crc)
{
crc = adler32(0L, Z_NULL, 0);
crc = adler32(crc, ((unsigned char *)send_buffer + (unsigned int)offset), bytes);
}
#endif
/* post the recv for the pong */
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
bytes, &actual_size, BMI_PRE_ALLOC, i,
NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
}
/* send the ping */
if (opts->test == EXPECTED) {
ret = BMI_post_send(&(op_id[SEND]), peer_addr, ((char *)send_buffer) + offset,
bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL);
} else {
ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr,
((char*)send_buffer) + offset, bytes, BMI_PRE_ALLOC, i,
NULL, *context, NULL);
}
if (ret < 0) {
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "send ping failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
/* complete the receive for the pong */
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
#ifdef HAVE_LIBZ
if(opts->crc && opts->test == EXPECTED)
{
rcrc = adler32(0L, Z_NULL, 0);
rcrc = adler32(rcrc, recv_buffer, bytes);
if(rcrc != crc)
{
fprintf(stderr, "CRC Mismatch! "
"Sent %llu but received %llu\n",
llu(crc),
llu(rcrc));
}
}
#endif
#ifdef WIN32
QueryPerformanceCounter(&end);
#else
gettimeofday(&end, NULL);
#endif
if (!warmup) {
#ifdef WIN32
val[i] = ((double) end.QuadPart - (double) start.QuadPart) / (double) freq.QuadPart;
#else
val[i] = (double) end.tv_sec +
(double) end.tv_usec * 0.000001;
val[i] -= (double) start.tv_sec +
(double) start.tv_usec * 0.000001;
#endif
lat += val[i];
}
}
if (!warmup) {
double stdev = 0.0;
lat = lat / (double) iterations * 1000000.0 / 2.0;
min = 999999.9;
max = 0.0;
avg = 0.0;
/* convert seconds to MB/s */
for (i=0; i < iterations; i++) {
val[i] = (double) bytes * 2 / val[i] / 1000000.0;
avg += val[i];
if (val[i] < min) min = val[i];
if (val[i] > max) max = val[i];
}
avg /= iterations;
if (iterations > 1) {
for (i=0; i < iterations; i++) {
double diff = val[i] - avg;
stdev += diff * diff;
}
stdev = sqrt(stdev / (iterations - 1));
}
fprintf(stdout, "%10d %12.3f %12.3f +- %9.3f %12.3f %12.3f\n", bytes, lat, avg, stdev, min, max);
lat = 0.0;
bytes *= 2;
run++;
} else warmup = 0;
}
/* free up the message buffers */
BMI_memfree(peer_addr, recv_buffer, max_bytes, BMI_RECV);
BMI_memfree(peer_addr, send_buffer, max_bytes, BMI_SEND);
return ret;
}
int main(
int argc,
char **argv)
{
int ret = -1;
int outcount = 0;
void *mybuffer;
PVFS_BMI_addr_t server_addr;
bmi_op_id_t op;
bmi_error_code_t error_code;
flow_descriptor *flow_d = NULL;
int i = 0;
bmi_size_t actual_size;
double time1, time2;
PINT_Request *req;
bmi_context_id context;
/*************************************************************/
/* initialization stuff */
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(
0, (GOSSIP_FLOW_PROTO_DEBUG | GOSSIP_BMI_DEBUG_TCP));
/* Dist init */
PINT_dist_initialize(NULL);
/* start up BMI */
ret = BMI_initialize("bmi_tcp", NULL, 0);
if (ret < 0)
{
fprintf(stderr, "BMI init failure.\n");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
fprintf(stderr, "BMI_open_context() failure.\n");
return (-1);
}
/* initialize the flow interface */
ret = PINT_flow_initialize("flowproto_multiqueue", 0);
if (ret < 0)
{
fprintf(stderr, "flow init failure.\n");
return (-1);
}
/* send some random crap to the other side to start up communication */
ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3335");
if (ret < 0)
{
fprintf(stderr, "BMI lookup failure.\n");
return (-1);
}
ret = BMI_post_sendunexpected(&op, server_addr, &mybuffer, 1,
BMI_EXT_ALLOC, 0, NULL, context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of request */
do
{
ret = BMI_test(op, &outcount, &error_code, &actual_size,
NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "Request send failed.\n");
if (ret < 0)
{
errno = -ret;
perror("BMI_test");
}
return (-1);
}
}
/******************************************************/
/* setup request/dist stuff */
/* request description */
/* just want one contiguous region */
ret = PVFS_Request_contiguous(TEST_SIZE, PVFS_BYTE, &req);
if (ret < 0)
{
fprintf(stderr, "PVFS_Request_contiguous() failure.\n");
return (-1);
}
/******************************************************/
/* setup communicaton stuff */
/* memory buffer to xfer */
mybuffer = (void *) malloc(TEST_SIZE);
if (!mybuffer)
{
fprintf(stderr, "mem.\n");
return (-1);
}
/* mark it so that we can check correctness */
for (i = 0; i < (TEST_SIZE / (sizeof(int))); i++)
{
((int *) mybuffer)[i] = i;
}
/* create a flow descriptor */
flow_d = PINT_flow_alloc();
if (!flow_d)
{
fprintf(stderr, "mem.\n");
return (-1);
}
/* file data */
flow_d->file_data.fsize = TEST_SIZE;
flow_d->file_data.server_nr = 0;
flow_d->file_data.server_ct = 1;
flow_d->file_data.extend_flag = 0;
flow_d->file_data.dist = PINT_dist_create("basic_dist");
if (!flow_d->file_data.dist)
{
fprintf(stderr, "Error: failed to create dist.\n");
return (-1);
}
ret = PINT_dist_lookup(flow_d->file_data.dist);
if (ret != 0)
{
fprintf(stderr, "Error: failed to lookup dist.\n");
return (-1);
}
flow_d->file_req = req;
flow_d->tag = 0;
flow_d->user_ptr = NULL;
flow_d->aggregate_size = TEST_SIZE;
/* fill in flow details */
flow_d->src.endpoint_id = MEM_ENDPOINT;
flow_d->src.u.mem.buffer = mybuffer;
flow_d->dest.endpoint_id = BMI_ENDPOINT;
flow_d->dest.u.bmi.address = server_addr;
/***************************************************
* test memory to bmi (analogous to client side write)
*/
time1 = Wtime();
ret = block_on_flow(flow_d);
if (ret < 0)
{
return (-1);
}
time2 = Wtime();
/*******************************************************/
/* final cleanup and output */
#if 0
printf("Client bw (send): %f MB/sec\n",
((TEST_SIZE) / ((time2 - time1) * 1000000.0)));
#endif
PINT_flow_free(flow_d);
/* shut down flow interface */
ret = PINT_flow_finalize();
if (ret < 0)
{
fprintf(stderr, "flow finalize failure.\n");
return (-1);
}
/* shut down BMI */
BMI_close_context(context);
BMI_finalize();
free(mybuffer);
gossip_disable();
return (0);
}
