int BMI_alloc_buffers(
struct mem_buffers *bufs,
int num_buffers,
int size,
PVFS_BMI_addr_t addr,
enum bmi_op_type send_recv)
{
int i = 0;
bufs->buffers = (void **) malloc(num_buffers * sizeof(void *));
if (!bufs->buffers)
{
return (-1);
}
for (i = 0; i < num_buffers; i++)
{
bufs->buffers[i] = (void *) BMI_memalloc(addr, size, send_recv);
if (!bufs->buffers[i])
{
return (-1);
}
}
bufs->num_buffers = num_buffers;
bufs->size = size;
return (0);
}
/*
* Used by both encode functions, request and response, to set
* up the one buffer which will hold the encoded message.
*/
static int
encode_common(struct PINT_encoded_msg *target_msg, int maxsize)
{
int ret = 0;
void *buf = NULL;
gossip_debug(GOSSIP_ENDECODE_DEBUG,"encode_common\n");
/* this encoder always uses just one buffer */
BF_ENCODE_TARGET_MSG_INIT(target_msg);
/* allocate the max size buffer to avoid the work of calculating it */
buf = (initializing_sizes ? malloc(maxsize) :
BMI_memalloc(target_msg->dest, maxsize, BMI_SEND));
if (!buf)
{
gossip_err("Error: failed to BMI_malloc memory for response.\n");
gossip_err("Error: is BMI address %llu still valid?\n", llu(target_msg->dest));
ret = -PVFS_ENOMEM;
goto out;
}
target_msg->buffer_list[0] = buf;
target_msg->alloc_size_list[0] = maxsize;
target_msg->ptr_current = buf;
/* generic header */
memcpy(target_msg->ptr_current, le_bytefield_table.generic_header,
PINT_ENC_GENERIC_HEADER_SIZE);
target_msg->ptr_current += PINT_ENC_GENERIC_HEADER_SIZE;
out:
return ret;
}
static PINT_sm_action test_recv_one_msg_f(
struct PINT_smcb *smcb, job_status_s *js_p)
{
struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
int ret;
gossip_debug(GOSSIP_SERVER_DEBUG, "\n\n%s: entry\n\n", __func__);
s_op->msgarray_op.msgpair.max_resp_sz = PINT_encode_calc_max_size(
PINT_ENCODE_RESP,
PVFS_SERV_WRITE_COMPLETION,
ENCODING_LE_BFIELD);
s_op->msgarray_op.msgpair.encoded_resp_p = BMI_memalloc(
s_op->addr, //s_op->msgarray_op.msgpair.svr_addr,
s_op->msgarray_op.msgpair.max_resp_sz,
BMI_RECV);
if (!s_op->msgarray_op.msgpair.encoded_resp_p)
{
gossip_err("BMI_memalloc (for write ack) failed\n");
return -PVFS_ENOMEM;
}
//gossip_debug(GOSSIP_LB_DEBUG, "bmi memalloc success\n");
/*
pre-post this recv with an infinite timeout and adjust it
after the flow completes since we don't know how long a flow
can take at this point
*/
ret = job_bmi_recv(
s_op->addr, //s_op->msgarray_op.msgpair.svr_addr,
s_op->msgarray_op.msgpair.encoded_resp_p,
s_op->msgarray_op.msgpair.max_resp_sz,
5,
BMI_PRE_ALLOC,
smcb,
IO_SM_PHASE_FINAL_ACK,
js_p,
&s_op->msgarray_op.msgpair.recv_id,
server_job_context,
JOB_TIMEOUT_INF, NULL);
if (ret < 0)
{
gossip_err("job_bmi_recv (write ack) failed\n");
}else{
gossip_debug(GOSSIP_LB_DEBUG, "job_bmi_recv correct:ret = %d\n", ret);
}
return ret;
// assert(ret == 0);
}
static PINT_sm_action small_io_start_job(
struct PINT_smcb *smcb, job_status_s *js_p)
{
struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
int ret;
job_id_t tmp_id;
PINT_Request_state * file_req_state;
PINT_request_file_data fdata;
PINT_Request_result result;
struct filesystem_configuration_s * fs_config;
struct server_configuration_s * server_config;
memset(&s_op->resp.u.small_io, 0, sizeof(struct PVFS_servresp_small_io));
/* set io type in response to io type in request. This is
* needed by the client so it konws how to decode the response
* appropriately.
*/
s_op->resp.u.small_io.io_type = s_op->req->u.small_io.io_type;
if(s_op->req->u.small_io.io_type == PVFS_IO_READ &&
s_op->ds_attr.u.datafile.b_size == 0)
{
/* nothing to read. return SM_ACTION_DEFERRED */
js_p->error_code = 0;
return SM_ACTION_COMPLETE;
}
file_req_state = PINT_new_request_state(
s_op->req->u.small_io.file_req);
fdata.server_nr = s_op->req->u.small_io.server_nr;
fdata.server_ct = s_op->req->u.small_io.server_ct;
fdata.dist = s_op->req->u.small_io.dist;
result.offset_array = s_op->u.small_io.offsets;
result.size_array = s_op->u.small_io.sizes;
result.segmax = IO_MAX_REGIONS;
result.bytemax = s_op->req->u.small_io.aggregate_size;
result.bytes = 0;
result.segs = 0;
PINT_REQUEST_STATE_SET_TARGET(file_req_state,
s_op->req->u.small_io.file_req_offset);
PINT_REQUEST_STATE_SET_FINAL(file_req_state,
s_op->req->u.small_io.file_req_offset +
s_op->req->u.small_io.aggregate_size);
s_op->resp.u.small_io.bstream_size = s_op->ds_attr.u.datafile.b_size;
fdata.fsize = s_op->ds_attr.u.datafile.b_size;
fdata.extend_flag =
(s_op->req->u.small_io.io_type == PVFS_IO_READ) ? 0 : 1;
/* calculate the offsets and sizes in the datafile for the read or write */
ret = PINT_process_request(
file_req_state,
NULL,
&fdata,
&result,
PINT_SERVER);
if(ret < 0)
{
gossip_err("small_io: Failed to process file request\n");
js_p->error_code = ret;
return SM_ACTION_COMPLETE;
}
/* figure out if the fs config has trove data sync turned on or off
*/
server_config = get_server_config_struct();
if(!server_config)
{
gossip_err("small_io: server config is NULL!\n");
js_p->error_code = -PVFS_EINVAL;
return SM_ACTION_COMPLETE;
}
fs_config = PINT_config_find_fs_id(
server_config, s_op->req->u.small_io.fs_id);
if(!fs_config)
{
gossip_err("small_io: Failed to get filesystem "
"config from fs_id of: %d\n",
s_op->req->u.small_io.fs_id);
js_p->error_code = -PVFS_EINVAL;
return SM_ACTION_COMPLETE;
}
if(s_op->req->u.small_io.io_type == PVFS_IO_WRITE)
{
ret = job_trove_bstream_write_list(
s_op->req->u.small_io.fs_id,
s_op->req->u.small_io.handle,
(char **)&s_op->req->u.small_io.buffer,
(TROVE_size *)&s_op->req->u.small_io.total_bytes,
1,
s_op->u.small_io.offsets,
s_op->u.small_io.sizes,
result.segs,
&s_op->resp.u.small_io.result_size,
(fs_config->trove_sync_data ? TROVE_SYNC : 0),
NULL,
smcb,
0,
js_p,
&tmp_id,
server_job_context,
s_op->req->hints);
if(ret < 0)
{
gossip_err("small_io: Failed to post trove bstream write\n");
}
}
else
{
/* allocate space for the read in the response buffer */
s_op->resp.u.small_io.buffer = BMI_memalloc(
s_op->addr, result.bytes, BMI_SEND);
if(!s_op->resp.u.small_io.buffer)
{
js_p->error_code = -PVFS_ENOMEM;
return SM_ACTION_COMPLETE;
}
s_op->u.small_io.result_bytes = result.bytes;
ret = job_trove_bstream_read_list(
s_op->req->u.small_io.fs_id,
s_op->req->u.small_io.handle,
(char **)&s_op->resp.u.small_io.buffer,
&s_op->u.small_io.result_bytes,
1,
s_op->u.small_io.offsets,
s_op->u.small_io.sizes,
result.segs,
&s_op->resp.u.small_io.result_size,
(fs_config->trove_sync_data ? TROVE_SYNC : 0),
NULL,
smcb,
0,
js_p,
&tmp_id,
server_job_context,
s_op->req->hints);
if(ret < 0)
{
gossip_err("small-io: Failed to post trove bstream read\n");
js_p->error_code = ret;
return SM_ACTION_COMPLETE;
}
}
PINT_free_request_state(file_req_state);
return ret;
}
int main(
int argc,
char **argv)
{
struct options *user_opts = NULL;
struct server_request *my_req = NULL;
struct server_ack *my_ack = NULL;
int ret = -1;
PVFS_BMI_addr_t client_addr;
void *recv_buffer1 = NULL;
void *recv_buffer2 = NULL;
bmi_op_id_t server_ops[2];
bmi_error_code_t error_code;
int outcount = 0;
struct BMI_unexpected_info request_info;
bmi_size_t actual_size;
bmi_size_t size_list[2];
void *buffer_list[2];
int last = 0;
int i = 0;
bmi_context_id context;
char method[24], *cp;
int len;
/* grab any command line options */
user_opts = parse_args(argc, argv);
if (!user_opts)
{
return (-1);
}
/* set debugging stuff */
gossip_enable_stderr();
gossip_set_debug_mask(0, GOSSIP_BMI_DEBUG_ALL);
/* convert address to bmi method type by prefixing bmi_ */
cp = strchr(user_opts->hostid, ':');
if (!cp)
return 1;
len = cp - user_opts->hostid;
strcpy(method, "bmi_");
strncpy(method + 4, user_opts->hostid, len);
method[4+len] = '\0';
/* initialize local interface (default options) */
ret = BMI_initialize(method, user_opts->hostid, BMI_INIT_SERVER);
if (ret < 0)
{
errno = -ret;
perror("BMI_initialize");
return (-1);
}
ret = BMI_open_context(&context);
if (ret < 0)
{
errno = -ret;
perror("BMI_open_context()");
return (-1);
}
/* wait for an initial request */
do
{
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0)
{
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return (-1);
}
if (request_info.error_code != 0)
{
fprintf(stderr, "Request recv failure (bad state).\n");
return (-1);
}
printf("Received a new request.\n");
if (request_info.size != sizeof(struct server_request))
{
fprintf(stderr, "Bad Request!\n");
exit(-1);
}
my_req = (struct server_request *) request_info.buffer;
client_addr = request_info.addr;
/* create an ack */
my_ack = (struct server_ack *) BMI_memalloc(client_addr,
sizeof(struct server_ack),
BMI_SEND);
if (!my_ack)
{
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
memset(my_ack, 0, sizeof(struct server_ack));
/* create 2 buffers to recv into */
recv_buffer1 = BMI_memalloc(client_addr, (my_req->size / 2), BMI_RECV);
recv_buffer2 = BMI_memalloc(client_addr,
(my_req->size - (my_req->size / 2)), BMI_RECV);
if (!recv_buffer1 || !recv_buffer2)
{
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
buffer_list[0] = recv_buffer1;
buffer_list[1] = recv_buffer2;
size_list[0] = my_req->size / 2;
size_list[1] = my_req->size - (my_req->size / 2);
/* post the ack */
ret = BMI_post_send(&(server_ops[1]), client_addr, my_ack,
sizeof(struct server_ack), BMI_PRE_ALLOC, 0, NULL,
context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_send_failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of ack send */
do
{
ret = BMI_test(server_ops[1], &outcount, &error_code,
&actual_size, NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "ack send failed.\n");
return (-1);
}
}
/* post the recv */
ret = BMI_post_recv_list(&(server_ops[0]), client_addr, buffer_list,
size_list, 2, my_req->size, &actual_size,
BMI_PRE_ALLOC, 0, NULL, context, NULL);
if (ret < 0)
{
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
}
if (ret == 0)
{
/* turning this into a blocking call for testing :) */
/* check for completion of data payload recv */
do
{
ret = BMI_test(server_ops[0], &outcount, &error_code,
&actual_size, NULL, 10, context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0)
{
fprintf(stderr, "data recv failed.\n");
return (-1);
}
}
if (actual_size != my_req->size)
{
printf("Short recv.\n");
printf("diff: %d\n", (int) (my_req->size - actual_size));
}
/* check validity of received message */
for (i = 0; i < ((my_req->size / 2) / sizeof(int)); i++)
{
if (((int *) recv_buffer1)[i] != i)
{
fprintf(stderr, "Validation failure, offset %d.\n", i);
}
}
last = i;
for (i = last;
i < (last + ((my_req->size - (my_req->size / 2)) / sizeof(int))); i++)
{
if (((int *) recv_buffer2)[i - last] != i)
{
fprintf(stderr, "Validation failure, offset %d.\n", i);
}
}
/* free up the message buffers */
BMI_memfree(client_addr, recv_buffer1, (my_req->size / 2), BMI_RECV);
BMI_memfree(client_addr, recv_buffer2, (my_req->size -
(my_req->size / 2)), BMI_RECV);
BMI_memfree(client_addr, my_ack, sizeof(struct server_ack), BMI_SEND);
BMI_unexpected_free(client_addr, my_req);
/* shutdown the local interface */
BMI_close_context(context);
ret = BMI_finalize();
if (ret < 0)
{
errno = -ret;
perror("BMI_finalize");
return (-1);
}
/* turn off debugging stuff */
gossip_disable();
free(user_opts->hostid);
free(user_opts);
return (0);
}
static int do_client(struct options *opts, bmi_context_id *context)
{
int ret = 0;
int i = 0;
PVFS_BMI_addr_t peer_addr;
void *recv_buffer = NULL;
void *send_buffer = NULL;
bmi_op_id_t op_id[2];
bmi_error_code_t error_code;
int outcount = 0;
bmi_size_t actual_size;
struct msg *tx_msg = NULL;
int bytes = MIN_BYTES;
int max_bytes = MAX_BYTES;
int warmup = 1;
int iterations = 0;
int msg_len = 0;
int run = 0;
#ifdef WIN32
LARGE_INTEGER start;
LARGE_INTEGER end;
#else
struct timeval start;
struct timeval end;
#endif
double *val = NULL;
double lat = 0.0;
double min = 99999.9;
double max = 0.0;
double avg = 0.0;
int offset;
#ifdef HAVE_LIBZ
unsigned long crc=0, rcrc=0;
#endif
/* get a bmi_addr for the server */
ret = BMI_addr_lookup(&peer_addr, opts->hostid);
if (ret < 0) {
errno = -ret;
perror("BMI_addr_lookup");
return (-1);
}
if (opts->test == UNEXPECTED) {
ret = BMI_get_info(peer_addr, BMI_GET_UNEXP_SIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
} else {
int maxsize = 0;
ret = BMI_get_info(peer_addr, BMI_CHECK_MAXSIZE,
(void *)&maxsize);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (maxsize < max_bytes) max_bytes = maxsize;
}
msg_len = sizeof(struct msg);
/* create send buffer */
send_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_SEND);
if (!send_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
if(opts->crc)
{
for(i = 0; i < max_bytes; ++i)
{
((char *)send_buffer)[i] = i;
}
}
else
{
memset(send_buffer, 0, max_bytes);
}
tx_msg = (struct msg *) send_buffer;
tx_msg->test = htonl(opts->test);
/* create a buffer to recv into */
recv_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_RECV);
if (!recv_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
/* post the test parameters */
ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr, tx_msg,
msg_len, BMI_PRE_ALLOC, 0, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data send failed.\n");
return (-1);
}
if (actual_size != msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
return (-1);
}
}
/* post a recv for the ack */
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
msg_len, &actual_size, BMI_PRE_ALLOC, 0, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
return (-1);
}
}
val = calloc(ITERATIONS, sizeof(double));
if (val == NULL) {
fprintf(stderr, "calloc() for val failed\n");
return -1;
}
/* make sure server has posted first recv */
#ifdef WIN32
Sleep(1000);
#else
sleep(1);
#endif
fprintf(stdout, " Bytes usecs MB/s StdDev Min Max\n");
/* start iterations */
while (bytes <= max_bytes) {
iterations = bytes_to_iterations(bytes);
for (i=0; i < iterations; i++) {
#ifdef WIN32
offset = rand() % (max_bytes - bytes - 1);
#else
offset = random() % (max_bytes - bytes - 1);
#endif
#ifdef WIN32
QueryPerformanceCounter(&start);
#else
gettimeofday(&start, NULL);
#endif
#ifdef HAVE_LIBZ
if(opts->crc)
{
crc = adler32(0L, Z_NULL, 0);
crc = adler32(crc, ((unsigned char *)send_buffer + (unsigned int)offset), bytes);
}
#endif
/* post the recv for the pong */
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
bytes, &actual_size, BMI_PRE_ALLOC, i,
NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
}
/* send the ping */
if (opts->test == EXPECTED) {
ret = BMI_post_send(&(op_id[SEND]), peer_addr, ((char *)send_buffer) + offset,
bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL);
} else {
ret = BMI_post_sendunexpected(&(op_id[SEND]), peer_addr,
((char*)send_buffer) + offset, bytes, BMI_PRE_ALLOC, i,
NULL, *context, NULL);
}
if (ret < 0) {
fprintf(stderr, "BMI_post_sendunexpected failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "send ping failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
/* complete the receive for the pong */
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
#ifdef HAVE_LIBZ
if(opts->crc && opts->test == EXPECTED)
{
rcrc = adler32(0L, Z_NULL, 0);
rcrc = adler32(rcrc, recv_buffer, bytes);
if(rcrc != crc)
{
fprintf(stderr, "CRC Mismatch! "
"Sent %llu but received %llu\n",
llu(crc),
llu(rcrc));
}
}
#endif
#ifdef WIN32
QueryPerformanceCounter(&end);
#else
gettimeofday(&end, NULL);
#endif
if (!warmup) {
#ifdef WIN32
val[i] = ((double) end.QuadPart - (double) start.QuadPart) / (double) freq.QuadPart;
#else
val[i] = (double) end.tv_sec +
(double) end.tv_usec * 0.000001;
val[i] -= (double) start.tv_sec +
(double) start.tv_usec * 0.000001;
#endif
lat += val[i];
}
}
if (!warmup) {
double stdev = 0.0;
lat = lat / (double) iterations * 1000000.0 / 2.0;
min = 999999.9;
max = 0.0;
avg = 0.0;
/* convert seconds to MB/s */
for (i=0; i < iterations; i++) {
val[i] = (double) bytes * 2 / val[i] / 1000000.0;
avg += val[i];
if (val[i] < min) min = val[i];
if (val[i] > max) max = val[i];
}
avg /= iterations;
if (iterations > 1) {
for (i=0; i < iterations; i++) {
double diff = val[i] - avg;
stdev += diff * diff;
}
stdev = sqrt(stdev / (iterations - 1));
}
fprintf(stdout, "%10d %12.3f %12.3f +- %9.3f %12.3f %12.3f\n", bytes, lat, avg, stdev, min, max);
lat = 0.0;
bytes *= 2;
run++;
} else warmup = 0;
}
/* free up the message buffers */
BMI_memfree(peer_addr, recv_buffer, max_bytes, BMI_RECV);
BMI_memfree(peer_addr, send_buffer, max_bytes, BMI_SEND);
return ret;
}
static int do_server(struct options *opts, bmi_context_id *context)
{
int ret = 0;
int i = 0;
PVFS_BMI_addr_t peer_addr;
PVFS_BMI_addr_t server_addr;
void *recv_buffer = NULL;
void *send_buffer = NULL;
bmi_op_id_t op_id[2];
bmi_error_code_t error_code;
int outcount = 0;
struct BMI_unexpected_info request_info;
bmi_size_t actual_size;
struct msg *tx_msg = NULL;
struct msg *rx_msg = NULL;
int bytes = MIN_BYTES;
int max_bytes = MAX_BYTES;
int warmup = 1;
int iterations = 0;
int msg_len = 0;
int run = 0;
/* wait for an initial request to get size */
do {
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return ret;
}
if (request_info.error_code != 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
return ret;
}
if (request_info.size != sizeof(struct msg)) {
fprintf(stderr, "Bad Request! Received %d bytes\n",
(int) request_info.size);
return ret;
}
rx_msg = (struct msg *) request_info.buffer;
opts->test = ntohl(rx_msg->test);
printf("Starting %s test%s\n", opts->test == EXPECTED ? "expected" : "unexpected",
opts->crc ? " with checksums" : "");
peer_addr = request_info.addr;
BMI_unexpected_free(peer_addr, request_info.buffer);
#ifdef WIN32
server_addr = (PVFS_BMI_addr_t) 0;
#endif
ret = BMI_get_info(server_addr, BMI_CHECK_MAXSIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (max_bytes > MAX_BYTES) max_bytes = MAX_BYTES;
if (opts->test == UNEXPECTED) {
ret = BMI_addr_lookup(&server_addr, opts->hostid);
if (ret < 0) {
errno = -ret;
perror("BMI_addr_lookup");
return (-1);
}
ret = BMI_get_info(server_addr, BMI_GET_UNEXP_SIZE,
(void *)&max_bytes);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
} else {
int maxsize = 0;
ret = BMI_get_info(server_addr, BMI_CHECK_MAXSIZE,
(void *)&maxsize);
if (ret < 0) {
fprintf(stderr, "BMI_get_info() returned %d\n", ret);
return ret;
}
if (maxsize < max_bytes) max_bytes = maxsize;
}
msg_len = sizeof(struct msg);
/* create an ack */
send_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_SEND);
if (!send_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
memset(send_buffer, 0, max_bytes);
tx_msg = (struct msg *) send_buffer;
tx_msg->test = htonl(opts->test);
/* create a buffer to recv into */
recv_buffer = BMI_memalloc(peer_addr, max_bytes, BMI_RECV);
if (!recv_buffer) {
fprintf(stderr, "BMI_memalloc failed.\n");
return (-1);
}
/* post the ack */
ret = BMI_post_send(&(op_id[SEND]), peer_addr, tx_msg,
msg_len, BMI_PRE_ALLOC, 0, NULL,
*context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_send failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "ack send failed.\n");
return (-1);
}
if (actual_size != (bmi_size_t) msg_len) {
fprintf(stderr, "Expected %d but received %llu\n",
msg_len, llu(actual_size));
}
}
/* start iterations */
while (bytes <= max_bytes) {
iterations = bytes_to_iterations(bytes);
for (i=0; i < iterations; i++) {
/* receive the ping */
if (opts->test == EXPECTED) {
ret = BMI_post_recv(&(op_id[RECV]), peer_addr, recv_buffer,
bytes, &actual_size, BMI_PRE_ALLOC, i, NULL,
*context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_recv_failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[RECV], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "data recv failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
} else { /* UNEXPECTED */
do {
ret = BMI_testunexpected(1, &outcount, &request_info, 10);
} while (ret == 0 && outcount == 0);
if (ret < 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
errno = -ret;
perror("BMI_testunexpected");
return ret;
}
if (request_info.error_code != 0) {
fprintf(stderr, "Request recv failure (bad state).\n");
return ret;
}
if (request_info.size != bytes) {
fprintf(stderr, "Bad Request! Received %d bytes\n",
(int) request_info.size);
return ret;
}
}
/* send the pong */
ret = BMI_post_send(&(op_id[SEND]), peer_addr,
(opts->test == EXPECTED ?
recv_buffer : request_info.buffer),
bytes, BMI_PRE_ALLOC, i, NULL, *context, NULL);
if (ret < 0) {
fprintf(stderr, "BMI_post_send failure.\n");
return (-1);
} else if (ret == 0) {
do {
ret = BMI_test(op_id[SEND], &outcount, &error_code,
&actual_size, NULL, 10, *context);
} while (ret == 0 && outcount == 0);
if (ret < 0 || error_code != 0) {
fprintf(stderr, "ack send failed.\n");
return (-1);
}
if (actual_size != bytes) {
fprintf(stderr, "Expected %d but received %llu\n",
bytes, llu(actual_size));
return (-1);
}
}
}
if (!warmup) {
bytes *= 2;
run++;
}
else warmup = 0;
}
/* free up the message buffers */
BMI_memfree(peer_addr, recv_buffer, max_bytes, BMI_RECV);
BMI_memfree(peer_addr, send_buffer, max_bytes, BMI_SEND);
return ret;
}
/* msgpairarray_post()
*
* The following elements of the PINT_sm_msgpair_state
* should be valid prior to this state (for each msgpair in array):
* - req (unencoded request)
* - srv_addr of each element in msg array
*
* This state performs the following operations for each msgpair,
* one at a time:
* (1) encodes request
* (2) calculates maximum response size
* (3) allocates BMI memory for response data (encoded)
* (4) gets a session tag for the pair of messages
* (5) posts the receive of the response
* (6) posts the send of the request
* (7) stores job ids for later matching
*
*/
static PINT_sm_action msgpairarray_post(
struct PINT_smcb *smcb, job_status_s *js_p)
{
PINT_sm_msgarray_op *mop = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
int ret = -PVFS_EINVAL, i = 0, tmp = 0;
struct server_configuration_s *server_config = NULL;
PVFS_msg_tag_t session_tag;
PINT_sm_msgpair_state *msg_p = NULL;
struct filesystem_configuration_s *cur_fs = NULL;
int must_loop_encodings = 0;
int local_enc_and_alloc = 0;
gossip_debug(
GOSSIP_MSGPAIR_DEBUG, "%s: sm %p "
"%d total message(s) with %d incomplete\n", __func__, smcb,
mop->count * 2, mop->params.comp_ct);
js_p->error_code = 0;
assert(mop->count > 0);
assert(mop->params.comp_ct >= 2);
for (i = 0; i < mop->count; i++)
{
msg_p = &mop->msgarray[i];
assert(msg_p);
/*
here we skip over the msgs that have already completed in
the case of being in the retry code path when it's ok
*/
if (msg_p->complete)
{
continue;
}
msg_p->op_status = 0;
if (msg_p->encoded_resp_p == NULL)
{
if (msg_p->fs_id != PVFS_FS_ID_NULL)
{
server_config = PINT_server_config_mgr_get_config(
msg_p->fs_id);
assert(server_config);
cur_fs = PINT_config_find_fs_id(
server_config, msg_p->fs_id);
PINT_server_config_mgr_put_config(server_config);
assert(cur_fs);
msg_p->enc_type = cur_fs->encoding;
}
if (!ENCODING_IS_VALID(msg_p->enc_type))
{
PRINT_ENCODING_ERROR("supported", msg_p->enc_type);
must_loop_encodings = 1;
msg_p->enc_type = (ENCODING_INVALID_MIN + 1);
}
else if (!ENCODING_IS_SUPPORTED(msg_p->enc_type))
{
PRINT_ENCODING_ERROR("supported", msg_p->enc_type);
must_loop_encodings = 1;
msg_p->enc_type = ENCODING_SUPPORTED_MIN;
}
try_next_encoding:
assert(ENCODING_IS_VALID(msg_p->enc_type));
ret = PINT_encode(&msg_p->req, PINT_ENCODE_REQ,
&msg_p->encoded_req, msg_p->svr_addr,
msg_p->enc_type);
if (ret != 0)
{
if (must_loop_encodings)
{
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "Looping through "
"encodings [%d/%d]\n", msg_p->enc_type,
ENCODING_INVALID_MAX);
msg_p->enc_type++;
if (ENCODING_IS_VALID(msg_p->enc_type))
{
goto try_next_encoding;
}
}
gossip_lerr("msgpairarray_post: PINT_encode failed\n");
js_p->error_code = ret;
return SM_ACTION_COMPLETE;
}
/* calculate max response msg size and allocate space */
msg_p->max_resp_sz = PINT_encode_calc_max_size(
PINT_ENCODE_RESP, msg_p->req.op, msg_p->enc_type);
msg_p->encoded_resp_p = BMI_memalloc(
msg_p->svr_addr, msg_p->max_resp_sz, BMI_RECV);
if (msg_p->encoded_resp_p == NULL)
{
js_p->error_code = -PVFS_ENOMEM;
return SM_ACTION_COMPLETE;
}
local_enc_and_alloc = 1;
}
session_tag = PINT_util_get_next_tag();
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: "
"posting recv\n", __func__, smcb, i);
/* post receive of response; job_id stored in recv_id */
ret = job_bmi_recv(msg_p->svr_addr,
msg_p->encoded_resp_p,
msg_p->max_resp_sz,
session_tag,
BMI_PRE_ALLOC,
smcb,
i,
&msg_p->recv_status,
&msg_p->recv_id,
mop->params.job_context,
mop->params.job_timeout,
msg_p->req.hints);
if (ret == 0)
{
/* perform a quick test to see if the recv failed before posting
* the send; if it reports an error quickly then we can save the
* confusion of sending a request for which we can't recv a
* response
*/
ret = job_test(msg_p->recv_id, &tmp, NULL,
&msg_p->recv_status, 0,
mop->params.job_context);
}
if ((ret < 0) || (ret == 1))
{
/* it is impossible for this recv to complete at this point
* without errors; we haven't sent the request yet!
*/
assert(ret < 0 || msg_p->recv_status.error_code != 0);
if (ret < 0)
{
PVFS_perror_gossip("Post of receive failed", ret);
}
else
{
PVFS_perror_gossip("Receive immediately failed",
msg_p->recv_status.error_code);
}
msg_p->recv_id = 0;
msg_p->send_id = 0;
/* mark send as bad too and don't post it */
msg_p->send_status.error_code = msg_p->recv_status.error_code;
msg_p->op_status = msg_p->recv_status.error_code;
mop->params.comp_ct -= 2;
if (local_enc_and_alloc)
{
PINT_encode_release(&msg_p->encoded_req, PINT_ENCODE_REQ);
BMI_memfree(msg_p->svr_addr,msg_p->encoded_resp_p,
msg_p->max_resp_sz, BMI_RECV);
msg_p->encoded_resp_p = NULL;
local_enc_and_alloc = 0;
}
/* continue to send other array entries if possible */
continue;
}
/* if we reach here, the recv has been posted without failure, but
* has not completed yet
*/
assert(ret == 0);
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: "
"posting send\n", __func__, smcb, i);
/* post send of request; job_id stored in send_id */
ret = job_bmi_send_list(msg_p->encoded_req.dest,
msg_p->encoded_req.buffer_list,
msg_p->encoded_req.size_list,
msg_p->encoded_req.list_count,
msg_p->encoded_req.total_size,
session_tag,
msg_p->encoded_req.buffer_type,
1,
smcb,
mop->count+i,
&msg_p->send_status,
&msg_p->send_id,
mop->params.job_context,
mop->params.job_timeout,
msg_p->req.hints);
if ((ret < 0) ||
((ret == 1) && (msg_p->send_status.error_code != 0)))
{
if (ret < 0)
{
PVFS_perror_gossip("Post of send failed", ret);
}
else
{
PVFS_perror_gossip("Send immediately failed",
msg_p->send_status.error_code);
}
gossip_err_unless_quiet("Send error: cancelling recv.\n");
job_bmi_cancel(msg_p->recv_id, mop->params.job_context);
/* we still have to wait for recv completion, so just decrement
* comp_ct by one and keep going
*/
msg_p->op_status = msg_p->send_status.error_code;
msg_p->send_id = 0;
mop->params.comp_ct--;
}
else if (ret == 1)
{
/* immediate completion */
msg_p->send_id = 0;
/* decrement our count, since send is already done. */
mop->params.comp_ct--;
}
/* else: successful post, no immediate completion */
}
if (mop->params.comp_ct == 0)
{
/* everything is completed already (could happen in some failure
* cases); jump straight to final completion function.
*/
js_p->error_code = MSGPAIRS_COMPLETE;
return SM_ACTION_COMPLETE;
}
/* we are still waiting on operations to complete, next state
* transition will handle them
*/
return SM_ACTION_DEFERRED;
}
int main(int argc, char **argv)
{
int ret = -1;
struct request_foo* req = NULL;
struct ack_foo* ack = NULL;
PVFS_BMI_addr_t server_addr;
job_status_s status1;
job_id_t tmp_id;
job_context_id context;
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(0, 0);
/* start the BMI interface */
ret = BMI_initialize("bmi_tcp", NULL, 0);
if(ret < 0)
{
fprintf(stderr, "BMI_initialize failure.\n");
return(-1);
}
ret = trove_initialize(
TROVE_METHOD_DBPF, NULL, "/tmp/pvfs2-test-space", 0);
if(ret < 0)
{
fprintf(stderr, "trove_initialize failure.\n");
return(-1);
}
/* start the job interface */
ret = job_initialize(0);
if(ret < 0)
{
fprintf(stderr, "job_initialize failure.\n");
return(-1);
}
ret = job_open_context(&context);
if(ret < 0)
{
fprintf(stderr, "job_open_context() failure.\n");
return(-1);
}
/* lookup the server to get a BMI style address for it */
ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3414");
if(ret < 0)
{
fprintf(stderr, "BMI_addr_lookup failure.\n");
return(-1);
}
/* allocate some buffers for the req and ack */
req = BMI_memalloc(server_addr, sizeof(struct request_foo),
BMI_SEND);
ack = BMI_memalloc(server_addr, sizeof(struct ack_foo),
BMI_RECV);
if(!ack || ! req)
{
fprintf(stderr, "BMI_memalloc failure.\n");
return(-1);
}
/* send a message */
ret = job_bmi_send(server_addr, req, sizeof(struct request_foo),
0, BMI_PRE_ALLOC, 1, NULL, 0, &status1, &tmp_id, context,
JOB_TIMEOUT_INF, NULL);
if(ret < 0)
{
fprintf(stderr, "job_bmi_send() failure.\n");
return(-1);
}
if(ret == 0)
{
int count = 0;
ret = job_test(tmp_id, &count, NULL, &status1, -1, context);
if(ret < 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "job failure.\n");
return(-1);
}
/* receive a message */
ret = job_bmi_recv(server_addr, ack, sizeof(struct ack_foo),
0, BMI_PRE_ALLOC, NULL, 0, &status1, &tmp_id, context,
JOB_TIMEOUT_INF, NULL);
if(ret < 0)
{
fprintf(stderr, "job_bmi_recv() failure.\n");
return(-1);
}
if(ret == 0)
{
int count = 0;
ret = job_test(tmp_id, &count, NULL, &status1, -1, context);
if(ret < 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "job failure.\n");
return(-1);
}
/* check the size */
if(status1.actual_size != sizeof(struct ack_foo))
{
fprintf(stderr, "short recv.\n");
return(-1);
}
/* free memory buffers */
BMI_memfree(server_addr, req, sizeof(struct request_foo),
BMI_SEND);
BMI_memfree(server_addr, ack, sizeof(struct ack_foo),
BMI_RECV);
/* shut down the interfaces */
job_close_context(context);
job_finalize();
BMI_finalize();
trove_finalize(TROVE_METHOD_DBPF);
return(0);
}
int main(int argc, char **argv)
{
int ret = -1;
struct ack_foo* ack = NULL;
job_status_s status1;
struct BMI_unexpected_info req_info;
job_id_t job_id;
int outcount;
job_id_t tmp_id;
job_context_id context;
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(0, 0);
/* start the BMI interface */
ret = BMI_initialize("bmi_tcp", "tcp://NULL:3414", BMI_INIT_SERVER);
if(ret < 0)
{
fprintf(stderr, "BMI_initialize failure.\n");
return(-1);
}
ret = trove_initialize(
TROVE_METHOD_DBPF, NULL, "/tmp/pvfs2-test-space", 0);
if(ret < 0)
{
fprintf(stderr, "trove_initialize failure.\n");
return(-1);
}
/* start the flow interface */
ret = PINT_flow_initialize("flowproto_multiqueue", 0);
if(ret < 0)
{
fprintf(stderr, "flow_init failure.\n");
return(-1);
}
/* start the job interface */
ret = job_initialize(0);
if(ret < 0)
{
fprintf(stderr, "job_initialize failure.\n");
return(-1);
}
ret = job_open_context(&context);
if(ret < 0)
{
fprintf(stderr, "job_open_context() failure.\n");
return(-1);
}
/* post a job for unexpected receive */
ret = job_bmi_unexp(&req_info, NULL, 0, &status1, &job_id, 0, context);
if(ret < 0)
{
fprintf(stderr, "job_bmi_unexp() failure.\n");
return(-1);
}
if(ret != 1)
{
#if 0
/* exercise testworld() interface, block indefinitely */
outcount = 1;
ret = job_testworld(&job_id, &outcount, NULL, &status1, -1);
if(ret < 0 || outcount == 0)
{
fprintf(stderr, "job_testworld() failure.\n");
return(-1);
}
/* alternatively, try out the testsome interface */
outcount = 1;
ret = job_testsome(&job_id, &outcount, &foo, NULL, &status1, -1);
if(ret < 0 || outcount == 0)
{
fprintf(stderr, "job_testsome() failure.\n");
return(-1);
}
#else
/* ... or maybe even give job_test() a whirl */
ret = job_test(job_id, &outcount, NULL, &status1, 5000, context);
if(ret < 0 || outcount == 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
#endif
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "Bad status in unexp recv.\n");
return(-1);
}
/* allocate a buffer for the ack */
ack = BMI_memalloc(req_info.addr, sizeof(struct ack_foo),
BMI_SEND);
if(!ack)
{
fprintf(stderr, "BMI_memalloc failure.\n");
return(-1);
}
/* send a message */
ret = job_bmi_send(req_info.addr, ack, sizeof(struct ack_foo),
0, BMI_PRE_ALLOC, 0, NULL, 0, &status1, &tmp_id, context,
JOB_TIMEOUT_INF, NULL);
if(ret < 0)
{
fprintf(stderr, "job_bmi_send() failure.\n");
return(-1);
}
if(ret == 0)
{
int count = 0;
ret = job_test(tmp_id, &count, NULL, &status1, -1, context);
if(ret < 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "job failure.\n");
return(-1);
}
BMI_memfree(req_info.addr, ack, sizeof(struct ack_foo), BMI_RECV);
BMI_unexpected_free(req_info.addr, req_info.buffer);
/* shut down the interfaces */
job_close_context(context);
job_finalize();
PINT_flow_finalize();
BMI_finalize();
trove_finalize(TROVE_METHOD_DBPF);
return(0);
}