int MPID_nem_mx_cancel_send(MPIDI_VC_t *vc, MPID_Request *sreq)
{
mx_request_t *mx_request = NULL;
mx_return_t ret;
uint32_t result;
int mpi_errno = MPI_SUCCESS;
int handled = FALSE;
if (!VC_CH(vc)->is_local)
{
mx_request = &(REQ_FIELD(sreq,mx_request));
ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
MPIU_ERR_CHKANDJUMP1(ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_cancel", "**mx_cancel %s", mx_strerror(ret));
if (result)
{
sreq->status.cancelled = TRUE;
sreq->cc = 0;
MPIU_Object_set_ref(sreq, 1);
MPID_nem_mx_pending_send_req--;
}
else
{
sreq->status.cancelled = FALSE;
}
handled = TRUE;
}
fn_exit:
return handled;
fn_fail:
goto fn_exit;
}
int MPID_nem_mx_cancel_recv(MPIDI_VC_t *vc, MPID_Request *rreq)
{
mx_request_t *mx_request = NULL;
mx_return_t ret;
uint32_t result;
int mpi_errno = MPI_SUCCESS;
int handled = FALSE;
mx_request = &(REQ_FIELD(rreq,mx_request));
/* FIXME this test is probably not correct with multiple netmods */
/* We need to know to which netmod a recv request actually "belongs" to */
if(mx_request != NULL)
{
ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
MPIU_ERR_CHKANDJUMP1(ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_cancel", "**mx_cancel %s", mx_strerror(ret));
if (result)
{
int found;
rreq->status.cancelled = TRUE;
found = MPIDI_CH3U_Recvq_DP(rreq);
MPIU_Assert(found);
rreq->status.count = 0;
MPID_REQUEST_SET_COMPLETED(rreq);
MPID_Request_release(rreq);
}
else
{
rreq->status.cancelled = FALSE;
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
handled = TRUE;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline void
receiver(mx_endpoint_t ep, int blocking, uint32_t match_val, uint32_t filter)
{
int count, len, iter, cur_req, num_req;
mx_status_t stat;
mx_request_t req[NUM_RREQ];
mx_request_t sreq;
mx_segment_t seg;
uint32_t result, usec;
struct timeval start_time, end_time;
double bw, pkts_per_sec;
char *buffer;
struct metadata info;
int bothways;
#if MX_THREAD_SAFE
struct mx_thread_arg args;
MX_THREAD_T thread;
#endif
uint64_t nic;
uint32_t eid;
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[0]);
/* wait for the receive to complete */
mx_test_or_wait(blocking, ep, &req[0], MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
if (filter != ~0) {
/* filter == ~0 means recv threads on master */
mx_decompose_endpoint_addr(stat.source, &nic, &eid);
mx_connect(ep, nic, eid, filter, MX_INFINITE, &stat.source);
}
len = ntohl(info.len);
iter = ntohl(info.iter);
Verify = ntohl(info.verify);
bothways = ntohl(info.bothways);
if (do_verbose)
printf("Starting test: len = %d, iter = %d\n", len, iter);
if (do_verbose && Verify) {
printf("Verifying results\n");
}
buffer = malloc(len * NUM_RREQ);
if (buffer == NULL) {
fprintf(stderr, "Can't allocate buffers\n");
exit(1);
}
if (bothways) {
#if MX_THREAD_SAFE
args.ep = ep;
args.dest = stat.source;
args.iter = iter;
args.len = len;
args.blocking = blocking;
num_threads++;
MX_THREAD_CREATE(&thread, &start_send_thread, &args);
#else
fprintf(stderr,"bothways not supported\n");
exit(1);
#endif
}
/* pre-post our receives */
num_req = NUM_RREQ;
if (num_req > iter)
num_req = iter;
for (cur_req = 0; cur_req < num_req; cur_req++) {
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0,
&req[cur_req]);
}
MX_MUTEX_LOCK(&stream_mutex);
++threads_running;
MX_MUTEX_UNLOCK(&stream_mutex);
while(threads_running != num_threads)
/* spin */;
#if DO_HANDSHAKE
/* post a send to let the sender know we are ready */
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
sreq = 0;
mx_isend(ep, &seg, 1, stat.source, match_val, NULL, &sreq);
mx_test_or_wait(blocking, ep, &sreq, MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "isend failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
#endif
/* start the test */
gettimeofday(&start_time, NULL);
for (count = 0; count < iter; count++) {
/* wait for the receive to complete */
cur_req = count & (NUM_RREQ - 1);
mx_test_or_wait(blocking, ep, &req[cur_req],
MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
if (stat.xfer_length != len) {
fprintf(stderr, "bad len %d != %d\n", stat.xfer_length, len);
exit(1);
}
/* hack since mx_cancel does not work */
if ((count + NUM_RREQ) > iter)
continue;
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
if (Verify)
mx_check_buffer(seg.segment_ptr, len);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0,
&req[cur_req]);
}
gettimeofday(&end_time, NULL);
usec = end_time.tv_usec - start_time.tv_usec;
usec += (end_time.tv_sec - start_time.tv_sec) * 1000000;
bw = ((double)iter * (double)len) / (double) usec;
pkts_per_sec = iter / ((double) usec / 1000000.0);
global_bwinfo.bandwidth = bw;
global_bwinfo.pkts_per_sec = pkts_per_sec;
/* printf("%8d %5.3f %5.3f\n", len, bw, pkts_per_sec);*/
#if 0 /* mx_cancel assert(0)'s */
for (cur_req = 0; cur_req < num_req; cur_req++) {
mx_cancel(ep, &req[cur_req]);
}
#endif
info.usec = htonl(usec);
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
sreq = 0;
mx_isend(ep, &seg, 1, stat.source, match_val, NULL, &sreq);
mx_test_or_wait(blocking, ep, &sreq, MX_INFINITE, &stat, &result);
free(buffer);
#if MX_THREAD_SAFE
if(bothways)
MX_THREAD_JOIN(thread);
#endif
}
