/**
* Initiate an asynchronous put.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
static int mca_btl_mx_put( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
mx_segment_t mx_segment[2];
mx_return_t mx_return;
uint32_t i = 0;
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
frag->endpoint = endpoint;
frag->type = MCA_BTL_MX_SEND;
descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
do {
mx_segment[i].segment_ptr = descriptor->des_src[i].seg_addr.pval;
mx_segment[i].segment_length = descriptor->des_src[i].seg_len;
} while (++i < descriptor->des_src_cnt);
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
endpoint->mx_peer_addr,
descriptor->des_dst[0].seg_key.key64, frag,
&frag->mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int mca_btl_mx_send( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* descriptor,
mca_btl_base_tag_t tag )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
mx_segment_t mx_segment[2];
mx_return_t mx_return;
uint64_t total_length = 0, tag64;
uint32_t i = 0;
int btl_ownership = (descriptor->des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
frag->endpoint = endpoint;
frag->type = MCA_BTL_MX_SEND;
do {
mx_segment[i].segment_ptr = descriptor->des_src[i].seg_addr.pval;
mx_segment[i].segment_length = descriptor->des_src[i].seg_len;
total_length += descriptor->des_src[i].seg_len;
} while (++i < descriptor->des_src_cnt);
tag64 = 0x01ULL | (((uint64_t)tag) << 8);
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
endpoint->mx_peer_addr,
tag64, frag, &frag->mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
#ifdef HAVE_MX_FORGET
{
uint32_t mx_result;
mx_return = mx_ibuffered( mx_btl->mx_endpoint, &(frag->mx_request), &mx_result );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_ibuffered failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_ERROR;
}
if( mx_result ) {
mx_return = mx_forget( mx_btl->mx_endpoint, &(frag->mx_request) );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
#endif
if( 2048 > total_length ) {
mx_status_t mx_status;
uint32_t mx_result;
/* let's check for completness */
mx_return = mx_test( mx_btl->mx_endpoint, &(frag->mx_request),
&mx_status, &mx_result );
if( OPAL_LIKELY(MX_SUCCESS == mx_return) ) {
if( mx_result ) {
if( MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags ) {
frag->base.des_cbfunc( &(mx_btl->super), frag->endpoint,
&(frag->base), OMPI_SUCCESS);
}
if( btl_ownership ) {
MCA_BTL_MX_FRAG_RETURN( mx_btl, frag );
}
return 1;
}
}
}
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
return OMPI_SUCCESS;
}
/**
* Initiate an inline send to the peer. If failure then return a descriptor.
*
* @param btl (IN) BTL module
* @param peer (IN) BTL peer addressing
*/
static int mca_btl_mx_sendi( struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct ompi_convertor_t* convertor,
void* header,
size_t header_size,
size_t payload_size,
uint8_t order,
uint32_t flags,
mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t** descriptor )
{
mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*) btl;
size_t max_data;
if( OPAL_UNLIKELY(MCA_BTL_MX_CONNECTED != ((mca_btl_mx_endpoint_t*)endpoint)->status) ) {
if( MCA_BTL_MX_NOT_REACHEABLE == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERROR;
if( MCA_BTL_MX_CONNECTION_PENDING == ((mca_btl_mx_endpoint_t*)endpoint)->status )
return OMPI_ERR_OUT_OF_RESOURCE;
if( OMPI_SUCCESS != mca_btl_mx_proc_connect( (mca_btl_mx_endpoint_t*)endpoint ) )
return OMPI_ERROR;
}
if( !ompi_convertor_need_buffers(convertor) ) {
uint32_t mx_segment_count = 0;
uint64_t tag64 = 0x01ULL | (((uint64_t)tag) << 8);
mx_return_t mx_return;
mx_request_t mx_request;
mx_segment_t mx_segments[2], *mx_segment = mx_segments;
if( 0 != header_size ) {
mx_segment->segment_ptr = header;
mx_segment->segment_length = header_size;
mx_segment++;
mx_segment_count++;
}
if( 0 != payload_size ) {
struct iovec iov;
uint32_t iov_count = 1;
iov.iov_base = NULL;
iov.iov_len = payload_size;
(void)ompi_convertor_pack( convertor, &iov, &iov_count, &max_data );
assert( max_data == payload_size );
mx_segment->segment_ptr = iov.iov_base;
mx_segment->segment_length = max_data;
mx_segment_count++;
}
mx_return = mx_isend( mx_btl->mx_endpoint, mx_segments, mx_segment_count,
endpoint->mx_peer_addr, tag64, NULL, &mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
return OMPI_ERROR;
}
#ifdef HAVE_MX_FORGET
{
uint32_t mx_result;
mx_return = mx_ibuffered( mx_btl->mx_endpoint, &mx_request, &mx_result );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_ibuffered failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
return OMPI_SUCCESS;
}
if( mx_result ) {
mx_return = mx_forget( mx_btl->mx_endpoint, &mx_request );
if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
opal_output( 0, "mx_forget failed with error %d (%s)\n",
mx_return, mx_strerror(mx_return) );
}
}
return OMPI_SUCCESS;
}
#endif
}
/* No optimization on this path. Just allocate a descriptor and return it
* to the user.
*/
*descriptor = mca_btl_mx_alloc( btl, endpoint, order,
header_size + payload_size, flags );
return OMPI_ERR_RESOURCE_BUSY;
}
static inline void
sender(mx_endpoint_t ep,mx_endpoint_addr_t dest, int iter, int len,
int blocking, int bothways, uint32_t match_val)
{
int count, cur_req, num_req;
mx_status_t stat;
mx_request_t req[NUM_SREQ];
mx_segment_t seg;
uint32_t result;
struct metadata info;
char *buffer;
uint32_t usec;
double bw, pkts_per_sec;
#if MX_THREAD_SAFE
MX_THREAD_T thread;
struct mx_thread_arg args;
#endif
struct bwinfo *bwinfo = NULL;
buffer = malloc(len * NUM_SREQ);
if (buffer == NULL) {
fprintf(stderr, "Can't allocate buffers\n");
exit(1);
}
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
info.len = htonl(len);
info.iter = htonl(iter);
info.verify = htonl(Verify);
info.bothways = htonl(bothways);
if (bothways) {
#if MX_THREAD_SAFE
args.ep = ep;
args.dest = dest;
args.iter = iter;
args.len = len;
args.blocking = blocking;
MX_THREAD_CREATE(&thread, &start_recv_thread, &args);
#else
fprintf(stderr, "bothways not supported\n");
exit(1);
#endif
}
mx_isend(ep, &seg, 1, dest, match_val, NULL, &req[0]);
/* wait for the send to complete */
mx_test_or_wait(blocking, ep, &req[0], MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "isend failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
MX_MUTEX_LOCK(&stream_mutex);
++threads_running;
MX_MUTEX_UNLOCK(&stream_mutex);
while(threads_running != num_threads)
/* spin */;
#if DO_HANDSHAKE
/* wait for the receiver to get ready */
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[0]);
/* wait for the receive to complete */
mx_test_or_wait(blocking, ep, &req[0], MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
#endif
num_req = NUM_SREQ;
if (num_req > iter)
num_req = iter;
for (cur_req = 0; cur_req < num_req; cur_req++) {
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
if (Verify)
mx_fill_buffer(seg.segment_ptr, len);
mx_isend(ep, &seg, 1, dest, match_val, NULL,
&req[cur_req]);
}
for (count = 0; count < iter; count++) {
/* wait for the send to complete */
cur_req = count & (NUM_SREQ - 1);
mx_test_or_wait(blocking, ep, &req[cur_req],
MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "isend failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
/* hack since mx_cancel does not work */
if ((count + NUM_SREQ) >= iter)
continue;
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
if (Verify)
mx_fill_buffer(seg.segment_ptr, len);
mx_isend(ep, &seg, 1, dest, match_val, NULL, &req[cur_req]);
}
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[0]);
/* wait for the receive to complete */
mx_test_or_wait(blocking, ep, &req[0], MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
usec = ntohl(info.usec);
bw = ((double)iter * (double)len) / (double) usec;
pkts_per_sec = iter / ((double) usec / 1000000.0);
if (match_val == MATCH_VAL_THREAD)
return;
if (match_val == MATCH_VAL_MAIN && bothways)
#if MX_THREAD_SAFE
if(bothways) {
printf("Send: %8d %5.3f %5.3f\n",
len, bw, pkts_per_sec);
MX_THREAD_JOIN(thread);
bwinfo = &global_bwinfo;
}
#endif
if (bwinfo) {
printf("Recv: %8d %5.3f %5.3f\n",
len, bwinfo->bandwidth, bwinfo->pkts_per_sec);
bw += bwinfo->bandwidth;
pkts_per_sec += bwinfo->pkts_per_sec;
}
printf("Total: %8d %5.3f %5.3f\n", len, bw, pkts_per_sec);
}
static inline void
receiver(mx_endpoint_t ep, int blocking, uint32_t match_val, uint32_t filter)
{
int count, len, iter, cur_req, num_req;
mx_status_t stat;
mx_request_t req[NUM_RREQ];
mx_request_t sreq;
mx_segment_t seg;
uint32_t result, usec;
struct timeval start_time, end_time;
double bw, pkts_per_sec;
char *buffer;
struct metadata info;
int bothways;
#if MX_THREAD_SAFE
struct mx_thread_arg args;
MX_THREAD_T thread;
#endif
uint64_t nic;
uint32_t eid;
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[0]);
/* wait for the receive to complete */
mx_test_or_wait(blocking, ep, &req[0], MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
if (filter != ~0) {
/* filter == ~0 means recv threads on master */
mx_decompose_endpoint_addr(stat.source, &nic, &eid);
mx_connect(ep, nic, eid, filter, MX_INFINITE, &stat.source);
}
len = ntohl(info.len);
iter = ntohl(info.iter);
Verify = ntohl(info.verify);
bothways = ntohl(info.bothways);
if (do_verbose)
printf("Starting test: len = %d, iter = %d\n", len, iter);
if (do_verbose && Verify) {
printf("Verifying results\n");
}
buffer = malloc(len * NUM_RREQ);
if (buffer == NULL) {
fprintf(stderr, "Can't allocate buffers\n");
exit(1);
}
if (bothways) {
#if MX_THREAD_SAFE
args.ep = ep;
args.dest = stat.source;
args.iter = iter;
args.len = len;
args.blocking = blocking;
num_threads++;
MX_THREAD_CREATE(&thread, &start_send_thread, &args);
#else
fprintf(stderr,"bothways not supported\n");
exit(1);
#endif
}
/* pre-post our receives */
num_req = NUM_RREQ;
if (num_req > iter)
num_req = iter;
for (cur_req = 0; cur_req < num_req; cur_req++) {
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0,
&req[cur_req]);
}
MX_MUTEX_LOCK(&stream_mutex);
++threads_running;
MX_MUTEX_UNLOCK(&stream_mutex);
while(threads_running != num_threads)
/* spin */;
#if DO_HANDSHAKE
/* post a send to let the sender know we are ready */
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
sreq = 0;
mx_isend(ep, &seg, 1, stat.source, match_val, NULL, &sreq);
mx_test_or_wait(blocking, ep, &sreq, MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "isend failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
#endif
/* start the test */
gettimeofday(&start_time, NULL);
for (count = 0; count < iter; count++) {
/* wait for the receive to complete */
cur_req = count & (NUM_RREQ - 1);
mx_test_or_wait(blocking, ep, &req[cur_req],
MX_INFINITE, &stat, &result);
if (!result) {
fprintf(stderr, "mx_wait failed\n");
exit(1);
}
if (stat.code != MX_STATUS_SUCCESS) {
fprintf(stderr, "irecv failed with status %s\n", mx_strstatus(stat.code));
exit(1);
}
if (stat.xfer_length != len) {
fprintf(stderr, "bad len %d != %d\n", stat.xfer_length, len);
exit(1);
}
/* hack since mx_cancel does not work */
if ((count + NUM_RREQ) > iter)
continue;
seg.segment_ptr = &buffer[cur_req * len];
seg.segment_length = len;
if (Verify)
mx_check_buffer(seg.segment_ptr, len);
mx_irecv(ep, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0,
&req[cur_req]);
}
gettimeofday(&end_time, NULL);
usec = end_time.tv_usec - start_time.tv_usec;
usec += (end_time.tv_sec - start_time.tv_sec) * 1000000;
bw = ((double)iter * (double)len) / (double) usec;
pkts_per_sec = iter / ((double) usec / 1000000.0);
global_bwinfo.bandwidth = bw;
global_bwinfo.pkts_per_sec = pkts_per_sec;
/* printf("%8d %5.3f %5.3f\n", len, bw, pkts_per_sec);*/
#if 0 /* mx_cancel assert(0)'s */
for (cur_req = 0; cur_req < num_req; cur_req++) {
mx_cancel(ep, &req[cur_req]);
}
#endif
info.usec = htonl(usec);
seg.segment_ptr = &info;
seg.segment_length = sizeof(info);
sreq = 0;
mx_isend(ep, &seg, 1, stat.source, match_val, NULL, &sreq);
mx_test_or_wait(blocking, ep, &sreq, MX_INFINITE, &stat, &result);
free(buffer);
#if MX_THREAD_SAFE
if(bothways)
MX_THREAD_JOIN(thread);
#endif
}
int
ompi_mtl_mx_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
mx_return_t mx_return;
uint64_t match_bits;
mca_mtl_mx_request_t mtl_mx_request;
size_t length;
mx_status_t mx_status;
uint32_t result;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
ompi_mtl_datatype_pack(convertor,
&mtl_mx_request.mx_segment[0].segment_ptr,
&length,
&mtl_mx_request.free_after);
mtl_mx_request.mx_segment[0].segment_length = length;
mtl_mx_request.convertor = convertor;
mtl_mx_request.type = OMPI_MTL_MX_ISEND;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"issend bits: 0x%016" PRIu64 "\n",
match_bits));
if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mx_return = mx_issend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_issend";
} else {
mx_return = mx_isend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request.mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
&mtl_mx_request,
&mtl_mx_request.mx_request
);
where = "mx_isend";
}
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n",
where, mx_strerror(mx_return), peer_name);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_ERROR;
}
do {
mx_return = mx_test(ompi_mtl_mx.mx_endpoint,
&mtl_mx_request.mx_request,
&mx_status,
&result);
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output, "Error in mx_wait (error %s)\n", mx_strerror(mx_return));
abort();
}
if( OPAL_UNLIKELY(result && mx_status.code != MX_STATUS_SUCCESS) ) {
opal_output(ompi_mtl_base_framework.framework_output,
"Error in ompi_mtl_mx_send, mx_wait returned something other than MX_STATUS_SUCCESS: mx_status.code = %d.\n",
mx_status.code);
abort();
}
} while(!result);
/* Free buffer if needed */
if(mtl_mx_request.free_after) {
free(mtl_mx_request.mx_segment[0].segment_ptr);
}
return OMPI_SUCCESS;
}
int
ompi_mtl_mx_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm,
int dest,
int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode,
bool blocking,
mca_mtl_request_t * mtl_request)
{
mx_return_t mx_return;
uint64_t match_bits;
mca_mtl_mx_request_t * mtl_mx_request = (mca_mtl_mx_request_t*) mtl_request;
size_t length;
ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, dest );
mca_mtl_mx_endpoint_t* mx_endpoint = (mca_mtl_mx_endpoint_t*) ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
char* where;
assert(mtl == &ompi_mtl_mx.super);
MX_SET_SEND_BITS(match_bits, comm->c_contextid, comm->c_my_rank, tag);
ompi_mtl_datatype_pack(convertor,
&mtl_mx_request->mx_segment[0].segment_ptr,
&length,
&mtl_mx_request->free_after);
mtl_mx_request->mx_segment[0].segment_length = length;
mtl_mx_request->convertor = convertor;
mtl_mx_request->type = OMPI_MTL_MX_ISEND;
OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output,
"issend bits: 0x%016" PRIu64 "\n", match_bits));
if(mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mx_return = mx_issend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request->mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
mtl_mx_request,
&mtl_mx_request->mx_request
);
where = "mx_issend";
} else {
mx_return = mx_isend( ompi_mtl_mx.mx_endpoint,
mtl_mx_request->mx_segment,
1,
mx_endpoint->mx_peer_addr,
match_bits,
mtl_mx_request,
&mtl_mx_request->mx_request
);
where = "mx_isend";
}
if( OPAL_UNLIKELY(mx_return != MX_SUCCESS) ) {
char peer_name[MX_MAX_HOSTNAME_LEN];
if(MX_SUCCESS != mx_nic_id_to_hostname( mx_endpoint->mx_peer->nic_id, peer_name)) {
sprintf( peer_name, "unknown %lx nic_id", (long)mx_endpoint->mx_peer->nic_id );
}
opal_output(ompi_mtl_base_framework.framework_output, "Error in %s (error %s) sending to %s\n",
where, mx_strerror(mx_return), peer_name);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
