void orte_qos_ack_channel_process_ack (int status, orte_process_name_t* sender, opal_buffer_t *buffer, orte_rml_tag_t tag, void *cbdata) { /* process ack received for the msg */ uint32_t num_msgs_acked, channel_num, i; int32_t num_values, room_num; orte_rml_send_t *msg, *missed_msg; void *occupant = NULL; orte_rml_channel_t *channel; orte_qos_ack_channel_t *ack_chan; uint32_t *seq_num_array; uint32_t ack_type; uint32_t missed_msg_seq_num = 0; num_values = 1; /* unpack channel number first */ opal_dss.unpack(buffer, (void*) &channel_num, &num_values, OPAL_UINT32); OPAL_OUTPUT_VERBOSE((5, orte_qos_base_framework.framework_output, "orte_qos_ack_channel_process_ack recieved ack on channel = %d", channel_num)); channel = orte_rml_base_get_channel (channel_num); if ((NULL != channel) || (NULL != channel->qos_channel_ptr)) { ack_chan = (orte_qos_ack_channel_t *) (channel->qos_channel_ptr); seq_num_array = malloc (sizeof(uint32_t) * ack_chan->window); num_values = 1; /* unpack ack type */ opal_dss.unpack(buffer, (void*) &ack_type, &num_values, OPAL_UINT32); num_values = 1; /* unpack num messages acked */ opal_dss.unpack(buffer, (void*) &num_msgs_acked, &num_values, OPAL_UINT32); OPAL_OUTPUT_VERBOSE((5, orte_qos_base_framework.framework_output, "orte_qos_ack_channel_process_ack recieved ack type %d for %d msgs on channel = %d", ack_type, num_msgs_acked, channel_num)); if (ACK_OUT_OF_ORDER != ack_type) { //handle normal ACK for (i = 0; i < num_msgs_acked; i++) { opal_dss.unpack(buffer, (void*) &seq_num_array[i], &num_values, OPAL_UINT32); room_num = orte_qos_ack_channel_get_msg_room (ack_chan, seq_num_array[i]); opal_hotel_checkout_and_return_occupant(&ack_chan->outstanding_msgs, room_num, &occupant); orte_qos_ack_channel_set_msg_room(ack_chan, seq_num_array[i], -1); if((occupant != NULL) && (room_num != -1)) { msg = (orte_rml_send_t*) occupant; OPAL_OUTPUT_VERBOSE((10, orte_rml_base_framework.framework_output, "Releasing sent message with tag %d and seq_num %d after receiving Ack from dest ", msg->tag, msg->seq_num )); msg->status = ORTE_SUCCESS; ORTE_RML_SEND_COMPLETE(msg); } else { OPAL_OUTPUT_VERBOSE((10, orte_rml_base_framework.framework_output, "OOPS received an ACK for already completed seq_num =%d ", seq_num_array[i] )); } } } else { // handle out of order ACK - complete msgs received in order, retry the lost msg. for (i = 0; i < num_msgs_acked; i++) { opal_dss.unpack(buffer, (void*) &seq_num_array[i], &num_values, OPAL_UINT32); room_num = orte_qos_ack_channel_get_msg_room (ack_chan, seq_num_array[i]); opal_hotel_checkout_and_return_occupant(&ack_chan->outstanding_msgs, room_num, &occupant); orte_qos_ack_channel_set_msg_room(ack_chan, seq_num_array[i], -1); if ((NULL != occupant) && ((i == 0 )|| (seq_num_array[i] == seq_num_array[i-1] +1 ))) { msg = (orte_rml_send_t*) occupant; msg->status = ORTE_SUCCESS; ORTE_RML_SEND_COMPLETE(msg); } else { if (NULL != occupant) { // num_missed_msgs = (seq_num_array[i] - seq_num_array [i-1] - 1); assert( i == num_msgs_acked -1); /* recheck the ith msg */ opal_hotel_checkin(&ack_chan->outstanding_msgs, (void*)occupant, &room_num); orte_qos_ack_channel_set_msg_room (ack_chan, seq_num_array[i], room_num); /* resend and recheck all the missed msgs*/ missed_msg_seq_num = seq_num_array[i-1] + 1; for (; missed_msg_seq_num < seq_num_array[i]; missed_msg_seq_num++) { room_num = orte_qos_ack_channel_get_msg_room (ack_chan, missed_msg_seq_num); opal_hotel_checkout_and_return_occupant (&ack_chan->outstanding_msgs, room_num, &occupant); assert ( NULL != occupant); missed_msg = (orte_rml_send_t*) occupant; missed_msg->status = ORTE_ERR_LOST_MSG_IN_WINDOW; opal_hotel_checkin(&ack_chan->outstanding_msgs, (void*)missed_msg, &room_num); orte_qos_ack_channel_set_msg_room (ack_chan, missed_msg_seq_num, room_num); /* send this out on wire directly */ ORTE_OOB_SEND (missed_msg); } //end for } else { OPAL_OUTPUT_VERBOSE((10, orte_rml_base_framework.framework_output, "OOPS received an ACK for already completed seq_num =%d ", seq_num_array[i] )); }//end if (NULL != occupant) } //end else } // end for }//end out of order ack processing free(seq_num_array); }else { OPAL_OUTPUT_VERBOSE((5, orte_qos_base_framework.framework_output, "orte_qos_ack_channel_msg_ack_recv_callback recieved ack on non existent channel = %d", channel_num)); } }
void pmix_server_keyval_client(int status, orte_process_name_t* sender, opal_buffer_t *buffer, orte_rml_tag_t tg, void *cbdata) { int rc, ret, room_num = -1; int32_t cnt; pmix_server_req_t *req=NULL; opal_list_t info; opal_value_t *iptr; opal_pmix_pdata_t *pdata; opal_process_name_t source; opal_output_verbose(1, orte_pmix_server_globals.output, "%s recvd lookup data return", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)); OBJ_CONSTRUCT(&info, opal_list_t); /* unpack the room number of the request tracker */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &room_num, &cnt, OPAL_INT))) { ORTE_ERROR_LOG(rc); return; } /* unpack the status */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &ret, &cnt, OPAL_INT))) { ORTE_ERROR_LOG(rc); ret = rc; goto release; } opal_output_verbose(5, orte_pmix_server_globals.output, "%s recvd lookup returned status %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ret); if (ORTE_SUCCESS == ret) { /* see if any data was included - not an error if the answer is no */ cnt = 1; while (OPAL_SUCCESS == opal_dss.unpack(buffer, &source, &cnt, OPAL_NAME)) { pdata = OBJ_NEW(opal_pmix_pdata_t); pdata->proc = source; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &iptr, &cnt, OPAL_VALUE))) { ORTE_ERROR_LOG(rc); OBJ_RELEASE(pdata); continue; } opal_output_verbose(5, orte_pmix_server_globals.output, "%s recvd lookup returned data %s of type %d from source %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), iptr->key, iptr->type, ORTE_NAME_PRINT(&source)); if (OPAL_SUCCESS != (rc = opal_value_xfer(&pdata->value, iptr))) { ORTE_ERROR_LOG(rc); OBJ_RELEASE(pdata); OBJ_RELEASE(iptr); continue; } OBJ_RELEASE(iptr); opal_list_append(&info, &pdata->super); } } release: if (0 <= room_num) { /* retrieve the tracker */ opal_hotel_checkout_and_return_occupant(&orte_pmix_server_globals.reqs, room_num, (void**)&req); } if (NULL != req) { /* pass down the response */ if (NULL != req->opcbfunc) { req->opcbfunc(ret, req->cbdata); } else if (NULL != req->lkcbfunc) { req->lkcbfunc(ret, &info, req->cbdata); } else { /* should not happen */ ORTE_ERROR_LOG(ORTE_ERR_NOT_SUPPORTED); } /* cleanup */ OPAL_LIST_DESTRUCT(&info); OBJ_RELEASE(req); } }
static inline int process_out_of_order_msg ( orte_qos_ack_channel_t *ack_chan, orte_rml_recv_t *msg) { int32_t rc, room_num, first_lost_msg_seq_num, num_lost_msgs, i; orte_rml_recv_t *out_msg; void *occupant = NULL; OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg msg %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), msg->seq_num)); /* if this msg is a duplicate - then do nothing */ if ((orte_qos_ack_channel_get_msg_room(ack_chan, msg->seq_num)) != -1) { OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg msg %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), msg->seq_num)); rc = ORTE_ERR_DUPLICATE_MSG; } else { if (OPAL_SUCCESS != (rc = opal_hotel_checkin(&ack_chan->outstanding_msgs, (void*)msg, &room_num))) { return rc; } OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "process_out_of_order_msg checked in msg %d in room %d\n", msg->seq_num, room_num)); orte_qos_ack_channel_set_msg_room (ack_chan, msg->seq_num, room_num); rc = ORTE_ERR_OUT_OF_ORDER_MSG; /* check if we need to send an ACK */ if (ack_chan->ack_msg_seq_num <= ack_chan->in_msg_seq_num) { OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg sending ack last seq_num = %d\n", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), msg->seq_num)); /* send ACK. */ send_ack (ack_chan, msg->channel_num, ACK_OUT_OF_ORDER, msg->seq_num); /* stop window ack timer */ opal_event_evtimer_del (&ack_chan->msg_ack_timer_event); } else { /* if we got a lost msg - any seq num between in_msg_seq_num and ack_seq_num*/ if (ack_chan->ack_msg_seq_num > msg->seq_num) { /* check if we have got all lost msgs */ first_lost_msg_seq_num = ack_chan->in_msg_seq_num + 1; num_lost_msgs = ack_chan->ack_msg_seq_num - ack_chan->in_msg_seq_num; OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg msg %d first_lost_msg =%d num_lost_msgs =%d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), msg->seq_num, first_lost_msg_seq_num, num_lost_msgs)); for (i =0 ; i < num_lost_msgs; i++) { if ((orte_qos_ack_channel_get_msg_room(ack_chan, first_lost_msg_seq_num +i)) == -1) break; } if (i == num_lost_msgs) { /* we got all the lost msgs so we can complete all the msgs in the hotel now */ /* reset ack_seq_num */ ack_chan->ack_msg_seq_num = first_lost_msg_seq_num -1; room_num = 0; for ( i = 0; room_num != -1; i++) { OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg got all lost msgs completing outstanding msgs %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (first_lost_msg_seq_num + i))); /* evict msg and complete it */ room_num = orte_qos_ack_channel_get_msg_room (ack_chan, first_lost_msg_seq_num +i); opal_hotel_checkout_and_return_occupant(&ack_chan->outstanding_msgs, room_num, &occupant); orte_qos_ack_channel_set_msg_room(ack_chan, first_lost_msg_seq_num +i, -1); out_msg = (orte_rml_recv_t *) occupant; if ((NULL != out_msg) && (room_num != -1)) { // set in seq num */ ack_chan->in_msg_seq_num = out_msg->seq_num; orte_rml_base_complete_recv_msg(&out_msg); /* completing recv msg to rml */ OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "process_out_of_order_msg completed recv msg %d", (first_lost_msg_seq_num + i))); } else { OPAL_OUTPUT_VERBOSE((1, orte_qos_base_framework.framework_output, "%s process_out_of_order_msg lost msg %d not in hotel", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (first_lost_msg_seq_num + i))); } } //end for /* send ACK */ send_ack (ack_chan, ack_chan->channel_num, ACK_RECV_MISSED_MSG, ack_chan->in_msg_seq_num); } //end if (i== num_lost_msgs) } // if (ack_chan->ack_msg_seq_num > msg->seq_num) } //end else } // end duplicate else return rc; }
static void pmix_server_dmdx_resp(int status, orte_process_name_t* sender, opal_buffer_t *buffer, orte_rml_tag_t tg, void *cbdata) { int rc, ret, room_num, rnum; int32_t cnt; opal_process_name_t target; pmix_server_req_t *req; datacaddy_t *d; opal_output_verbose(2, orte_pmix_server_globals.output, "%s dmdx:recv response from proc %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(sender)); /* unpack the status */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &ret, &cnt, OPAL_INT))) { ORTE_ERROR_LOG(rc); return; } /* unpack the id of the target whose info we just received */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &target, &cnt, OPAL_NAME))) { ORTE_ERROR_LOG(rc); return; } /* unpack our tracking room number */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &room_num, &cnt, OPAL_INT))) { ORTE_ERROR_LOG(rc); return; } /* unload the remainder of the buffer */ d = OBJ_NEW(datacaddy_t); if (OPAL_SUCCESS != (rc = opal_dss.unload(buffer, (void**)&d->data, &d->ndata))) { ORTE_ERROR_LOG(rc); return; } /* check the request out of the tracking hotel */ opal_hotel_checkout_and_return_occupant(&orte_pmix_server_globals.reqs, room_num, (void**)&req); /* return the returned data to the requestor */ if (NULL != req) { if (NULL != req->mdxcbfunc) { OBJ_RETAIN(d); req->mdxcbfunc(ret, d->data, d->ndata, req->cbdata, relcbfunc, d); } OBJ_RELEASE(req); } /* now see if anyone else was waiting for data from this target */ for (rnum=0; rnum < orte_pmix_server_globals.reqs.num_rooms; rnum++) { opal_hotel_knock(&orte_pmix_server_globals.reqs, rnum, (void**)&req); if (NULL == req) { continue; } if (req->target.jobid == target.jobid && req->target.vpid == target.vpid) { if (NULL != req->mdxcbfunc) { OBJ_RETAIN(d); req->mdxcbfunc(ret, d->data, d->ndata, req->cbdata, relcbfunc, d); } opal_hotel_checkout(&orte_pmix_server_globals.reqs, rnum); OBJ_RELEASE(req); } } OBJ_RELEASE(d); // maintain accounting }