static PINT_sm_action test_recv_one_msg_f(
struct PINT_smcb *smcb, job_status_s *js_p)
{
struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
int ret;
gossip_debug(GOSSIP_SERVER_DEBUG, "\n\n%s: entry\n\n", __func__);
s_op->msgarray_op.msgpair.max_resp_sz = PINT_encode_calc_max_size(
PINT_ENCODE_RESP,
PVFS_SERV_WRITE_COMPLETION,
ENCODING_LE_BFIELD);
s_op->msgarray_op.msgpair.encoded_resp_p = BMI_memalloc(
s_op->addr, //s_op->msgarray_op.msgpair.svr_addr,
s_op->msgarray_op.msgpair.max_resp_sz,
BMI_RECV);
if (!s_op->msgarray_op.msgpair.encoded_resp_p)
{
gossip_err("BMI_memalloc (for write ack) failed\n");
return -PVFS_ENOMEM;
}
//gossip_debug(GOSSIP_LB_DEBUG, "bmi memalloc success\n");
/*
pre-post this recv with an infinite timeout and adjust it
after the flow completes since we don't know how long a flow
can take at this point
*/
ret = job_bmi_recv(
s_op->addr, //s_op->msgarray_op.msgpair.svr_addr,
s_op->msgarray_op.msgpair.encoded_resp_p,
s_op->msgarray_op.msgpair.max_resp_sz,
5,
BMI_PRE_ALLOC,
smcb,
IO_SM_PHASE_FINAL_ACK,
js_p,
&s_op->msgarray_op.msgpair.recv_id,
server_job_context,
JOB_TIMEOUT_INF, NULL);
if (ret < 0)
{
gossip_err("job_bmi_recv (write ack) failed\n");
}else{
gossip_debug(GOSSIP_LB_DEBUG, "job_bmi_recv correct:ret = %d\n", ret);
}
return ret;
// assert(ret == 0);
}
/* msgpairarray_post()
*
* The following elements of the PINT_sm_msgpair_state
* should be valid prior to this state (for each msgpair in array):
* - req (unencoded request)
* - srv_addr of each element in msg array
*
* This state performs the following operations for each msgpair,
* one at a time:
* (1) encodes request
* (2) calculates maximum response size
* (3) allocates BMI memory for response data (encoded)
* (4) gets a session tag for the pair of messages
* (5) posts the receive of the response
* (6) posts the send of the request
* (7) stores job ids for later matching
*
*/
static PINT_sm_action msgpairarray_post(
struct PINT_smcb *smcb, job_status_s *js_p)
{
PINT_sm_msgarray_op *mop = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
int ret = -PVFS_EINVAL, i = 0, tmp = 0;
struct server_configuration_s *server_config = NULL;
PVFS_msg_tag_t session_tag;
PINT_sm_msgpair_state *msg_p = NULL;
struct filesystem_configuration_s *cur_fs = NULL;
int must_loop_encodings = 0;
int local_enc_and_alloc = 0;
gossip_debug(
GOSSIP_MSGPAIR_DEBUG, "%s: sm %p "
"%d total message(s) with %d incomplete\n", __func__, smcb,
mop->count * 2, mop->params.comp_ct);
js_p->error_code = 0;
assert(mop->count > 0);
assert(mop->params.comp_ct >= 2);
for (i = 0; i < mop->count; i++)
{
msg_p = &mop->msgarray[i];
assert(msg_p);
/*
here we skip over the msgs that have already completed in
the case of being in the retry code path when it's ok
*/
if (msg_p->complete)
{
continue;
}
msg_p->op_status = 0;
if (msg_p->encoded_resp_p == NULL)
{
if (msg_p->fs_id != PVFS_FS_ID_NULL)
{
server_config = PINT_server_config_mgr_get_config(
msg_p->fs_id);
assert(server_config);
cur_fs = PINT_config_find_fs_id(
server_config, msg_p->fs_id);
PINT_server_config_mgr_put_config(server_config);
assert(cur_fs);
msg_p->enc_type = cur_fs->encoding;
}
if (!ENCODING_IS_VALID(msg_p->enc_type))
{
PRINT_ENCODING_ERROR("supported", msg_p->enc_type);
must_loop_encodings = 1;
msg_p->enc_type = (ENCODING_INVALID_MIN + 1);
}
else if (!ENCODING_IS_SUPPORTED(msg_p->enc_type))
{
PRINT_ENCODING_ERROR("supported", msg_p->enc_type);
must_loop_encodings = 1;
msg_p->enc_type = ENCODING_SUPPORTED_MIN;
}
try_next_encoding:
assert(ENCODING_IS_VALID(msg_p->enc_type));
ret = PINT_encode(&msg_p->req, PINT_ENCODE_REQ,
&msg_p->encoded_req, msg_p->svr_addr,
msg_p->enc_type);
if (ret != 0)
{
if (must_loop_encodings)
{
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "Looping through "
"encodings [%d/%d]\n", msg_p->enc_type,
ENCODING_INVALID_MAX);
msg_p->enc_type++;
if (ENCODING_IS_VALID(msg_p->enc_type))
{
goto try_next_encoding;
}
}
gossip_lerr("msgpairarray_post: PINT_encode failed\n");
js_p->error_code = ret;
return SM_ACTION_COMPLETE;
}
/* calculate max response msg size and allocate space */
msg_p->max_resp_sz = PINT_encode_calc_max_size(
PINT_ENCODE_RESP, msg_p->req.op, msg_p->enc_type);
msg_p->encoded_resp_p = BMI_memalloc(
msg_p->svr_addr, msg_p->max_resp_sz, BMI_RECV);
if (msg_p->encoded_resp_p == NULL)
{
js_p->error_code = -PVFS_ENOMEM;
return SM_ACTION_COMPLETE;
}
local_enc_and_alloc = 1;
}
session_tag = PINT_util_get_next_tag();
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: "
"posting recv\n", __func__, smcb, i);
/* post receive of response; job_id stored in recv_id */
ret = job_bmi_recv(msg_p->svr_addr,
msg_p->encoded_resp_p,
msg_p->max_resp_sz,
session_tag,
BMI_PRE_ALLOC,
smcb,
i,
&msg_p->recv_status,
&msg_p->recv_id,
mop->params.job_context,
mop->params.job_timeout,
msg_p->req.hints);
if (ret == 0)
{
/* perform a quick test to see if the recv failed before posting
* the send; if it reports an error quickly then we can save the
* confusion of sending a request for which we can't recv a
* response
*/
ret = job_test(msg_p->recv_id, &tmp, NULL,
&msg_p->recv_status, 0,
mop->params.job_context);
}
if ((ret < 0) || (ret == 1))
{
/* it is impossible for this recv to complete at this point
* without errors; we haven't sent the request yet!
*/
assert(ret < 0 || msg_p->recv_status.error_code != 0);
if (ret < 0)
{
PVFS_perror_gossip("Post of receive failed", ret);
}
else
{
PVFS_perror_gossip("Receive immediately failed",
msg_p->recv_status.error_code);
}
msg_p->recv_id = 0;
msg_p->send_id = 0;
/* mark send as bad too and don't post it */
msg_p->send_status.error_code = msg_p->recv_status.error_code;
msg_p->op_status = msg_p->recv_status.error_code;
mop->params.comp_ct -= 2;
if (local_enc_and_alloc)
{
PINT_encode_release(&msg_p->encoded_req, PINT_ENCODE_REQ);
BMI_memfree(msg_p->svr_addr,msg_p->encoded_resp_p,
msg_p->max_resp_sz, BMI_RECV);
msg_p->encoded_resp_p = NULL;
local_enc_and_alloc = 0;
}
/* continue to send other array entries if possible */
continue;
}
/* if we reach here, the recv has been posted without failure, but
* has not completed yet
*/
assert(ret == 0);
gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: "
"posting send\n", __func__, smcb, i);
/* post send of request; job_id stored in send_id */
ret = job_bmi_send_list(msg_p->encoded_req.dest,
msg_p->encoded_req.buffer_list,
msg_p->encoded_req.size_list,
msg_p->encoded_req.list_count,
msg_p->encoded_req.total_size,
session_tag,
msg_p->encoded_req.buffer_type,
1,
smcb,
mop->count+i,
&msg_p->send_status,
&msg_p->send_id,
mop->params.job_context,
mop->params.job_timeout,
msg_p->req.hints);
if ((ret < 0) ||
((ret == 1) && (msg_p->send_status.error_code != 0)))
{
if (ret < 0)
{
PVFS_perror_gossip("Post of send failed", ret);
}
else
{
PVFS_perror_gossip("Send immediately failed",
msg_p->send_status.error_code);
}
gossip_err_unless_quiet("Send error: cancelling recv.\n");
job_bmi_cancel(msg_p->recv_id, mop->params.job_context);
/* we still have to wait for recv completion, so just decrement
* comp_ct by one and keep going
*/
msg_p->op_status = msg_p->send_status.error_code;
msg_p->send_id = 0;
mop->params.comp_ct--;
}
else if (ret == 1)
{
/* immediate completion */
msg_p->send_id = 0;
/* decrement our count, since send is already done. */
mop->params.comp_ct--;
}
/* else: successful post, no immediate completion */
}
if (mop->params.comp_ct == 0)
{
/* everything is completed already (could happen in some failure
* cases); jump straight to final completion function.
*/
js_p->error_code = MSGPAIRS_COMPLETE;
return SM_ACTION_COMPLETE;
}
/* we are still waiting on operations to complete, next state
* transition will handle them
*/
return SM_ACTION_DEFERRED;
}
int main(int argc, char **argv)
{
int ret = -1;
struct request_foo* req = NULL;
struct ack_foo* ack = NULL;
PVFS_BMI_addr_t server_addr;
job_status_s status1;
job_id_t tmp_id;
job_context_id context;
/* set debugging level */
gossip_enable_stderr();
gossip_set_debug_mask(0, 0);
/* start the BMI interface */
ret = BMI_initialize("bmi_tcp", NULL, 0);
if(ret < 0)
{
fprintf(stderr, "BMI_initialize failure.\n");
return(-1);
}
ret = trove_initialize(
TROVE_METHOD_DBPF, NULL, "/tmp/pvfs2-test-space", 0);
if(ret < 0)
{
fprintf(stderr, "trove_initialize failure.\n");
return(-1);
}
/* start the job interface */
ret = job_initialize(0);
if(ret < 0)
{
fprintf(stderr, "job_initialize failure.\n");
return(-1);
}
ret = job_open_context(&context);
if(ret < 0)
{
fprintf(stderr, "job_open_context() failure.\n");
return(-1);
}
/* lookup the server to get a BMI style address for it */
ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3414");
if(ret < 0)
{
fprintf(stderr, "BMI_addr_lookup failure.\n");
return(-1);
}
/* allocate some buffers for the req and ack */
req = BMI_memalloc(server_addr, sizeof(struct request_foo),
BMI_SEND);
ack = BMI_memalloc(server_addr, sizeof(struct ack_foo),
BMI_RECV);
if(!ack || ! req)
{
fprintf(stderr, "BMI_memalloc failure.\n");
return(-1);
}
/* send a message */
ret = job_bmi_send(server_addr, req, sizeof(struct request_foo),
0, BMI_PRE_ALLOC, 1, NULL, 0, &status1, &tmp_id, context,
JOB_TIMEOUT_INF, NULL);
if(ret < 0)
{
fprintf(stderr, "job_bmi_send() failure.\n");
return(-1);
}
if(ret == 0)
{
int count = 0;
ret = job_test(tmp_id, &count, NULL, &status1, -1, context);
if(ret < 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "job failure.\n");
return(-1);
}
/* receive a message */
ret = job_bmi_recv(server_addr, ack, sizeof(struct ack_foo),
0, BMI_PRE_ALLOC, NULL, 0, &status1, &tmp_id, context,
JOB_TIMEOUT_INF, NULL);
if(ret < 0)
{
fprintf(stderr, "job_bmi_recv() failure.\n");
return(-1);
}
if(ret == 0)
{
int count = 0;
ret = job_test(tmp_id, &count, NULL, &status1, -1, context);
if(ret < 0)
{
fprintf(stderr, "job_test() failure.\n");
return(-1);
}
}
/* check status */
if(status1.error_code != 0)
{
fprintf(stderr, "job failure.\n");
return(-1);
}
/* check the size */
if(status1.actual_size != sizeof(struct ack_foo))
{
fprintf(stderr, "short recv.\n");
return(-1);
}
/* free memory buffers */
BMI_memfree(server_addr, req, sizeof(struct request_foo),
BMI_SEND);
BMI_memfree(server_addr, ack, sizeof(struct ack_foo),
BMI_RECV);
/* shut down the interfaces */
job_close_context(context);
job_finalize();
BMI_finalize();
trove_finalize(TROVE_METHOD_DBPF);
return(0);
}
