static PINT_sm_action test_recv_one_msg_f( struct PINT_smcb *smcb, job_status_s *js_p) { struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret; gossip_debug(GOSSIP_SERVER_DEBUG, "\n\n%s: entry\n\n", __func__); s_op->msgarray_op.msgpair.max_resp_sz = PINT_encode_calc_max_size( PINT_ENCODE_RESP, PVFS_SERV_WRITE_COMPLETION, ENCODING_LE_BFIELD); s_op->msgarray_op.msgpair.encoded_resp_p = BMI_memalloc( s_op->addr, //s_op->msgarray_op.msgpair.svr_addr, s_op->msgarray_op.msgpair.max_resp_sz, BMI_RECV); if (!s_op->msgarray_op.msgpair.encoded_resp_p) { gossip_err("BMI_memalloc (for write ack) failed\n"); return -PVFS_ENOMEM; } //gossip_debug(GOSSIP_LB_DEBUG, "bmi memalloc success\n"); /* pre-post this recv with an infinite timeout and adjust it after the flow completes since we don't know how long a flow can take at this point */ ret = job_bmi_recv( s_op->addr, //s_op->msgarray_op.msgpair.svr_addr, s_op->msgarray_op.msgpair.encoded_resp_p, s_op->msgarray_op.msgpair.max_resp_sz, 5, BMI_PRE_ALLOC, smcb, IO_SM_PHASE_FINAL_ACK, js_p, &s_op->msgarray_op.msgpair.recv_id, server_job_context, JOB_TIMEOUT_INF, NULL); if (ret < 0) { gossip_err("job_bmi_recv (write ack) failed\n"); }else{ gossip_debug(GOSSIP_LB_DEBUG, "job_bmi_recv correct:ret = %d\n", ret); } return ret; // assert(ret == 0); }
/* msgpairarray_post() * * The following elements of the PINT_sm_msgpair_state * should be valid prior to this state (for each msgpair in array): * - req (unencoded request) * - srv_addr of each element in msg array * * This state performs the following operations for each msgpair, * one at a time: * (1) encodes request * (2) calculates maximum response size * (3) allocates BMI memory for response data (encoded) * (4) gets a session tag for the pair of messages * (5) posts the receive of the response * (6) posts the send of the request * (7) stores job ids for later matching * */ static PINT_sm_action msgpairarray_post( struct PINT_smcb *smcb, job_status_s *js_p) { PINT_sm_msgarray_op *mop = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL, i = 0, tmp = 0; struct server_configuration_s *server_config = NULL; PVFS_msg_tag_t session_tag; PINT_sm_msgpair_state *msg_p = NULL; struct filesystem_configuration_s *cur_fs = NULL; int must_loop_encodings = 0; int local_enc_and_alloc = 0; gossip_debug( GOSSIP_MSGPAIR_DEBUG, "%s: sm %p " "%d total message(s) with %d incomplete\n", __func__, smcb, mop->count * 2, mop->params.comp_ct); js_p->error_code = 0; assert(mop->count > 0); assert(mop->params.comp_ct >= 2); for (i = 0; i < mop->count; i++) { msg_p = &mop->msgarray[i]; assert(msg_p); /* here we skip over the msgs that have already completed in the case of being in the retry code path when it's ok */ if (msg_p->complete) { continue; } msg_p->op_status = 0; if (msg_p->encoded_resp_p == NULL) { if (msg_p->fs_id != PVFS_FS_ID_NULL) { server_config = PINT_server_config_mgr_get_config( msg_p->fs_id); assert(server_config); cur_fs = PINT_config_find_fs_id( server_config, msg_p->fs_id); PINT_server_config_mgr_put_config(server_config); assert(cur_fs); msg_p->enc_type = cur_fs->encoding; } if (!ENCODING_IS_VALID(msg_p->enc_type)) { PRINT_ENCODING_ERROR("supported", msg_p->enc_type); must_loop_encodings = 1; msg_p->enc_type = (ENCODING_INVALID_MIN + 1); } else if (!ENCODING_IS_SUPPORTED(msg_p->enc_type)) { PRINT_ENCODING_ERROR("supported", msg_p->enc_type); must_loop_encodings = 1; msg_p->enc_type = ENCODING_SUPPORTED_MIN; } try_next_encoding: assert(ENCODING_IS_VALID(msg_p->enc_type)); ret = PINT_encode(&msg_p->req, PINT_ENCODE_REQ, &msg_p->encoded_req, msg_p->svr_addr, msg_p->enc_type); if (ret != 0) { if (must_loop_encodings) { gossip_debug(GOSSIP_MSGPAIR_DEBUG, "Looping through " "encodings [%d/%d]\n", msg_p->enc_type, ENCODING_INVALID_MAX); msg_p->enc_type++; if (ENCODING_IS_VALID(msg_p->enc_type)) { goto try_next_encoding; } } gossip_lerr("msgpairarray_post: PINT_encode failed\n"); js_p->error_code = ret; return SM_ACTION_COMPLETE; } /* calculate max response msg size and allocate space */ msg_p->max_resp_sz = PINT_encode_calc_max_size( PINT_ENCODE_RESP, msg_p->req.op, msg_p->enc_type); msg_p->encoded_resp_p = BMI_memalloc( msg_p->svr_addr, msg_p->max_resp_sz, BMI_RECV); if (msg_p->encoded_resp_p == NULL) { js_p->error_code = -PVFS_ENOMEM; return SM_ACTION_COMPLETE; } local_enc_and_alloc = 1; } session_tag = PINT_util_get_next_tag(); gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: " "posting recv\n", __func__, smcb, i); /* post receive of response; job_id stored in recv_id */ ret = job_bmi_recv(msg_p->svr_addr, msg_p->encoded_resp_p, msg_p->max_resp_sz, session_tag, BMI_PRE_ALLOC, smcb, i, &msg_p->recv_status, &msg_p->recv_id, mop->params.job_context, mop->params.job_timeout, msg_p->req.hints); if (ret == 0) { /* perform a quick test to see if the recv failed before posting * the send; if it reports an error quickly then we can save the * confusion of sending a request for which we can't recv a * response */ ret = job_test(msg_p->recv_id, &tmp, NULL, &msg_p->recv_status, 0, mop->params.job_context); } if ((ret < 0) || (ret == 1)) { /* it is impossible for this recv to complete at this point * without errors; we haven't sent the request yet! */ assert(ret < 0 || msg_p->recv_status.error_code != 0); if (ret < 0) { PVFS_perror_gossip("Post of receive failed", ret); } else { PVFS_perror_gossip("Receive immediately failed", msg_p->recv_status.error_code); } msg_p->recv_id = 0; msg_p->send_id = 0; /* mark send as bad too and don't post it */ msg_p->send_status.error_code = msg_p->recv_status.error_code; msg_p->op_status = msg_p->recv_status.error_code; mop->params.comp_ct -= 2; if (local_enc_and_alloc) { PINT_encode_release(&msg_p->encoded_req, PINT_ENCODE_REQ); BMI_memfree(msg_p->svr_addr,msg_p->encoded_resp_p, msg_p->max_resp_sz, BMI_RECV); msg_p->encoded_resp_p = NULL; local_enc_and_alloc = 0; } /* continue to send other array entries if possible */ continue; } /* if we reach here, the recv has been posted without failure, but * has not completed yet */ assert(ret == 0); gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p msgpair %d: " "posting send\n", __func__, smcb, i); /* post send of request; job_id stored in send_id */ ret = job_bmi_send_list(msg_p->encoded_req.dest, msg_p->encoded_req.buffer_list, msg_p->encoded_req.size_list, msg_p->encoded_req.list_count, msg_p->encoded_req.total_size, session_tag, msg_p->encoded_req.buffer_type, 1, smcb, mop->count+i, &msg_p->send_status, &msg_p->send_id, mop->params.job_context, mop->params.job_timeout, msg_p->req.hints); if ((ret < 0) || ((ret == 1) && (msg_p->send_status.error_code != 0))) { if (ret < 0) { PVFS_perror_gossip("Post of send failed", ret); } else { PVFS_perror_gossip("Send immediately failed", msg_p->send_status.error_code); } gossip_err_unless_quiet("Send error: cancelling recv.\n"); job_bmi_cancel(msg_p->recv_id, mop->params.job_context); /* we still have to wait for recv completion, so just decrement * comp_ct by one and keep going */ msg_p->op_status = msg_p->send_status.error_code; msg_p->send_id = 0; mop->params.comp_ct--; } else if (ret == 1) { /* immediate completion */ msg_p->send_id = 0; /* decrement our count, since send is already done. */ mop->params.comp_ct--; } /* else: successful post, no immediate completion */ } if (mop->params.comp_ct == 0) { /* everything is completed already (could happen in some failure * cases); jump straight to final completion function. */ js_p->error_code = MSGPAIRS_COMPLETE; return SM_ACTION_COMPLETE; } /* we are still waiting on operations to complete, next state * transition will handle them */ return SM_ACTION_DEFERRED; }
int main(int argc, char **argv) { int ret = -1; struct request_foo* req = NULL; struct ack_foo* ack = NULL; PVFS_BMI_addr_t server_addr; job_status_s status1; job_id_t tmp_id; job_context_id context; /* set debugging level */ gossip_enable_stderr(); gossip_set_debug_mask(0, 0); /* start the BMI interface */ ret = BMI_initialize("bmi_tcp", NULL, 0); if(ret < 0) { fprintf(stderr, "BMI_initialize failure.\n"); return(-1); } ret = trove_initialize( TROVE_METHOD_DBPF, NULL, "/tmp/pvfs2-test-space", 0); if(ret < 0) { fprintf(stderr, "trove_initialize failure.\n"); return(-1); } /* start the job interface */ ret = job_initialize(0); if(ret < 0) { fprintf(stderr, "job_initialize failure.\n"); return(-1); } ret = job_open_context(&context); if(ret < 0) { fprintf(stderr, "job_open_context() failure.\n"); return(-1); } /* lookup the server to get a BMI style address for it */ ret = BMI_addr_lookup(&server_addr, "tcp://localhost:3414"); if(ret < 0) { fprintf(stderr, "BMI_addr_lookup failure.\n"); return(-1); } /* allocate some buffers for the req and ack */ req = BMI_memalloc(server_addr, sizeof(struct request_foo), BMI_SEND); ack = BMI_memalloc(server_addr, sizeof(struct ack_foo), BMI_RECV); if(!ack || ! req) { fprintf(stderr, "BMI_memalloc failure.\n"); return(-1); } /* send a message */ ret = job_bmi_send(server_addr, req, sizeof(struct request_foo), 0, BMI_PRE_ALLOC, 1, NULL, 0, &status1, &tmp_id, context, JOB_TIMEOUT_INF, NULL); if(ret < 0) { fprintf(stderr, "job_bmi_send() failure.\n"); return(-1); } if(ret == 0) { int count = 0; ret = job_test(tmp_id, &count, NULL, &status1, -1, context); if(ret < 0) { fprintf(stderr, "job_test() failure.\n"); return(-1); } } /* check status */ if(status1.error_code != 0) { fprintf(stderr, "job failure.\n"); return(-1); } /* receive a message */ ret = job_bmi_recv(server_addr, ack, sizeof(struct ack_foo), 0, BMI_PRE_ALLOC, NULL, 0, &status1, &tmp_id, context, JOB_TIMEOUT_INF, NULL); if(ret < 0) { fprintf(stderr, "job_bmi_recv() failure.\n"); return(-1); } if(ret == 0) { int count = 0; ret = job_test(tmp_id, &count, NULL, &status1, -1, context); if(ret < 0) { fprintf(stderr, "job_test() failure.\n"); return(-1); } } /* check status */ if(status1.error_code != 0) { fprintf(stderr, "job failure.\n"); return(-1); } /* check the size */ if(status1.actual_size != sizeof(struct ack_foo)) { fprintf(stderr, "short recv.\n"); return(-1); } /* free memory buffers */ BMI_memfree(server_addr, req, sizeof(struct request_foo), BMI_SEND); BMI_memfree(server_addr, ack, sizeof(struct ack_foo), BMI_RECV); /* shut down the interfaces */ job_close_context(context); job_finalize(); BMI_finalize(); trove_finalize(TROVE_METHOD_DBPF); return(0); }