int main(int argc, char *argv[])
{
#if OPAL_HAVE_POSIX_THREADS
int tid;
pthread_t *th;
#endif
if (argc != 2) {
printf("*** Incorrect number of arguments. Skipping test\n");
return 77;
}
nthreads = atoi(argv[1]);
/* first test single-threaded functionality */
/* -- cmpset 32-bit tests -- */
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 1);
opal_atomic_rmb();
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_32(&vol32, old32, new32) == 0);
opal_atomic_rmb();
assert(vol32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 1);
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_acq_32(&vol32, old32, new32) == 0);
assert(vol32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 1);
opal_atomic_rmb();
assert(vol32 == new32);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_cmpset_rel_32(&vol32, old32, new32) == 0);
opal_atomic_rmb();
assert(vol32 == 42);
/* -- cmpset 64-bit tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64
vol64 = 42, old64 = 42, new64 = 50;
assert(1 == opal_atomic_cmpset_64(&vol64, old64, new64));
opal_atomic_rmb();
assert(new64 == vol64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_64(&vol64, old64, new64) == 0);
opal_atomic_rmb();
assert(vol64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 1);
assert(vol64 == new64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_acq_64(&vol64, old64, new64) == 0);
assert(vol64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 1);
opal_atomic_rmb();
assert(vol64 == new64);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_cmpset_rel_64(&vol64, old64, new64) == 0);
opal_atomic_rmb();
assert(vol64 == 42);
#endif
/* -- cmpset int tests -- */
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset(&volint, oldint, newint) == 1);
opal_atomic_rmb();
assert(volint ==newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 1);
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset_acq(&volint, oldint, newint) == 0);
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 1);
opal_atomic_rmb();
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_cmpset_rel(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
/* -- cmpset ptr tests -- */
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 1);
opal_atomic_rmb();
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_ptr(&volptr, oldptr, newptr) == 0);
opal_atomic_rmb();
assert(volptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 1);
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_acq_ptr(&volptr, oldptr, newptr) == 0);
assert(volptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 1);
opal_atomic_rmb();
assert(volptr == newptr);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_cmpset_rel_ptr(&volptr, oldptr, newptr) == 0);
opal_atomic_rmb();
assert(volptr == (void *) 42);
/* -- add_32 tests -- */
val32 = 42;
assert(opal_atomic_add_32(&val32, 5) == (42 + 5));
opal_atomic_rmb();
assert((42 + 5) == val32);
/* -- add_64 tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64
val64 = 42;
assert(opal_atomic_add_64(&val64, 5) == (42 + 5));
opal_atomic_rmb();
assert((42 + 5) == val64);
#endif
/* -- add_int tests -- */
valint = 42;
opal_atomic_add(&valint, 5);
opal_atomic_rmb();
assert((42 + 5) == valint);
/* threaded tests */
val32 = 0;
#if OPAL_HAVE_ATOMIC_MATH_64
val64 = 0ul;
#endif
valint = 0;
/* -- create the thread set -- */
#if OPAL_HAVE_POSIX_THREADS
th = (pthread_t *) malloc(nthreads * sizeof(pthread_t));
if (!th) {
perror("malloc");
exit(EXIT_FAILURE);
}
for (tid = 0; tid < nthreads; tid++) {
if (pthread_create(&th[tid], NULL, thread_main, (void *) (unsigned long) tid) != 0) {
perror("pthread_create");
exit(EXIT_FAILURE);
}
}
/* -- wait for the thread set to finish -- */
for (tid = 0; tid < nthreads; tid++) {
void *thread_return;
if (pthread_join(th[tid], &thread_return) != 0) {
perror("pthread_join");
exit(EXIT_FAILURE);
}
}
free(th);
opal_atomic_rmb();
assert((5 * nthreads * nreps) == val32);
#if OPAL_HAVE_ATOMIC_MATH_64
opal_atomic_rmb();
assert((5 * nthreads * nreps) == val64);
#endif
opal_atomic_rmb();
assert((5 * nthreads * nreps) == valint);
#endif
return 0;
}
/* Setup eager RDMA buffers and notify the remote endpoint*/
void mca_btl_openib_endpoint_connect_eager_rdma(
mca_btl_openib_endpoint_t* endpoint)
{
mca_btl_openib_module_t* openib_btl = endpoint->endpoint_btl;
char *buf;
mca_btl_openib_recv_frag_t *headers_buf;
int i;
uint32_t flag = MCA_MPOOL_FLAGS_CACHE_BYPASS;
/* Set local rdma pointer to 1 temporarily so other threads will not try
* to enter the function */
if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL,
(void*)1))
return;
headers_buf = (mca_btl_openib_recv_frag_t*)
malloc(sizeof(mca_btl_openib_recv_frag_t) *
mca_btl_openib_component.eager_rdma_num);
if(NULL == headers_buf)
goto unlock_rdma_local;
#if HAVE_DECL_IBV_ACCESS_SO
/* Solaris implements the Relaxed Ordering feature defined in the
PCI Specification. With this in mind any memory region which
relies on a buffer being written in a specific order, for
example the eager rdma connections created in this routinue,
must set a strong order flag when registering the memory for
rdma operations.
The following flag will be interpreted and the appropriate
steps will be taken when the memory is registered in
openib_reg_mr(). */
flag |= MCA_MPOOL_FLAGS_SO_MEM;
#endif
buf = (char *) openib_btl->super.btl_mpool->mpool_alloc(openib_btl->super.btl_mpool,
openib_btl->eager_rdma_frag_size *
mca_btl_openib_component.eager_rdma_num,
mca_btl_openib_component.buffer_alignment,
flag,
(mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg);
if(!buf)
goto free_headers_buf;
buf = buf + openib_btl->eager_rdma_frag_size -
sizeof(mca_btl_openib_footer_t) - openib_btl->super.btl_eager_limit -
sizeof(mca_btl_openib_header_t);
for(i = 0; i < mca_btl_openib_component.eager_rdma_num; i++) {
opal_free_list_item_t *item;
mca_btl_openib_recv_frag_t * frag;
mca_btl_openib_frag_init_data_t init_data;
item = (opal_free_list_item_t*)&headers_buf[i];
item->registration = (mca_mpool_base_registration_t *)endpoint->eager_rdma_local.reg;
item->ptr = buf + i * openib_btl->eager_rdma_frag_size;
OBJ_CONSTRUCT(item, mca_btl_openib_recv_frag_t);
init_data.order = mca_btl_openib_component.credits_qp;
init_data.list = NULL;
mca_btl_openib_frag_init(item, &init_data);
frag = to_recv_frag(item);
to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA;
to_com_frag(frag)->endpoint = endpoint;
frag->ftr = (mca_btl_openib_footer_t*)
((char*)to_base_frag(frag)->segment.seg_addr.pval +
mca_btl_openib_component.eager_limit);
MCA_BTL_OPENIB_RDMA_MAKE_REMOTE(frag->ftr);
}
endpoint->eager_rdma_local.frags = headers_buf;
endpoint->eager_rdma_local.rd_win =
mca_btl_openib_component.eager_rdma_num >> 2;
endpoint->eager_rdma_local.rd_win =
endpoint->eager_rdma_local.rd_win?endpoint->eager_rdma_local.rd_win:1;
/* set local rdma pointer to real value */
(void)opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval,
(void*)1, buf);
if(mca_btl_openib_endpoint_send_eager_rdma(endpoint) == OPAL_SUCCESS) {
mca_btl_openib_device_t *device = endpoint->endpoint_btl->device;
mca_btl_openib_endpoint_t **p;
OBJ_RETAIN(endpoint);
assert(((opal_object_t*)endpoint)->obj_reference_count == 2);
do {
p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count];
} while(!opal_atomic_cmpset_ptr(p, NULL, endpoint));
OPAL_THREAD_ADD32(&openib_btl->eager_rdma_channels, 1);
/* from this point progress function starts to poll new buffer */
OPAL_THREAD_ADD32(&device->eager_rdma_buffers_count, 1);
return;
}
openib_btl->super.btl_mpool->mpool_free(openib_btl->super.btl_mpool,
buf, (mca_mpool_base_registration_t*)endpoint->eager_rdma_local.reg);
free_headers_buf:
free(headers_buf);
unlock_rdma_local:
/* set local rdma pointer back to zero. Will retry later */
(void)opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval,
endpoint->eager_rdma_local.base.pval, NULL);
endpoint->eager_rdma_local.frags = NULL;
}
/* Setup eager RDMA buffers and notify the remote endpoint*/
void mca_btl_wv_endpoint_connect_eager_rdma(
mca_btl_wv_endpoint_t* endpoint)
{
mca_btl_wv_module_t* wv_btl = endpoint->endpoint_btl;
char *buf;
mca_btl_wv_recv_frag_t *headers_buf;
int i;
/* Set local rdma pointer to 1 temporarily so other threads will not try
* to enter the function */
if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL,
(void*)1))
return;
headers_buf = (mca_btl_wv_recv_frag_t*)
malloc(sizeof(mca_btl_wv_recv_frag_t) *
mca_btl_wv_component.eager_rdma_num);
if(NULL == headers_buf)
goto unlock_rdma_local;
buf = (char *) wv_btl->super.btl_mpool->mpool_alloc(wv_btl->super.btl_mpool,
wv_btl->eager_rdma_frag_size *
mca_btl_wv_component.eager_rdma_num,
mca_btl_wv_component.buffer_alignment,
MCA_MPOOL_FLAGS_CACHE_BYPASS,
(mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg);
if(!buf)
goto free_headers_buf;
buf = buf + wv_btl->eager_rdma_frag_size -
sizeof(mca_btl_wv_footer_t) - wv_btl->super.btl_eager_limit -
sizeof(mca_btl_wv_header_t);
for(i = 0; i < mca_btl_wv_component.eager_rdma_num; i++) {
ompi_free_list_item_t *item;
mca_btl_wv_recv_frag_t * frag;
mca_btl_wv_frag_init_data_t init_data;
item = (ompi_free_list_item_t*)&headers_buf[i];
item->registration = (mca_mpool_base_registration_t *)endpoint->eager_rdma_local.reg;
item->ptr = buf + i * wv_btl->eager_rdma_frag_size;
OBJ_CONSTRUCT(item, mca_btl_wv_recv_frag_t);
init_data.order = mca_btl_wv_component.credits_qp;
init_data.list = NULL;
mca_btl_wv_frag_init(item, &init_data);
frag = to_recv_frag(item);
to_base_frag(frag)->type = MCA_BTL_WV_FRAG_EAGER_RDMA;
to_com_frag(frag)->endpoint = endpoint;
frag->ftr = (mca_btl_wv_footer_t*)
((char*)to_base_frag(frag)->segment.base.seg_addr.pval +
mca_btl_wv_component.eager_limit);
MCA_BTL_WV_RDMA_MAKE_REMOTE(frag->ftr);
}
endpoint->eager_rdma_local.frags = headers_buf;
endpoint->eager_rdma_local.rd_win =
mca_btl_wv_component.eager_rdma_num >> 2;
endpoint->eager_rdma_local.rd_win =
endpoint->eager_rdma_local.rd_win?endpoint->eager_rdma_local.rd_win:1;
/* set local rdma pointer to real value */
opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, (void*)1,
buf);
if(mca_btl_wv_endpoint_send_eager_rdma(endpoint) == OMPI_SUCCESS) {
mca_btl_wv_device_t *device = endpoint->endpoint_btl->device;
mca_btl_wv_endpoint_t **p;
OBJ_RETAIN(endpoint);
assert(((opal_object_t*)endpoint)->obj_reference_count == 2);
do {
p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count];
} while(!opal_atomic_cmpset_ptr(p, NULL, endpoint));
OPAL_THREAD_ADD32(&wv_btl->eager_rdma_channels, 1);
/* from this point progress function starts to poll new buffer */
OPAL_THREAD_ADD32(&device->eager_rdma_buffers_count, 1);
return;
}
wv_btl->super.btl_mpool->mpool_free(wv_btl->super.btl_mpool,
buf, (mca_mpool_base_registration_t*)endpoint->eager_rdma_local.reg);
free_headers_buf:
free(headers_buf);
unlock_rdma_local:
/* set local rdma pointer back to zero. Will retry later */
opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval,
endpoint->eager_rdma_local.base.pval, NULL);
endpoint->eager_rdma_local.frags = NULL;
}
static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
bool pval_clean = false;
int qp;
/* If the CPC has an endpoint_finalize function, call it */
if (NULL != endpoint->endpoint_local_cpc->cbm_endpoint_finalize) {
endpoint->endpoint_local_cpc->cbm_endpoint_finalize(endpoint);
}
/* Release CTS buffer */
opal_btl_openib_connect_base_free_cts(endpoint);
/* Release memory resources */
do {
/* Make sure that mca_btl_openib_endpoint_connect_eager_rdma ()
* was not in "connect" or "bad" flow (failed to allocate memory)
* and changed the pointer back to NULL
*/
if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL,
(void*)1)) {
if ((void*)1 != endpoint->eager_rdma_local.base.pval &&
NULL != endpoint->eager_rdma_local.base.pval) {
endpoint->endpoint_btl->super.btl_mpool->mpool_free(endpoint->endpoint_btl->super.btl_mpool,
endpoint->eager_rdma_local.base.pval,
(mca_mpool_base_registration_t*)endpoint->eager_rdma_local.reg);
pval_clean=true;
}
} else {
pval_clean=true;
}
} while (!pval_clean);
/* Close opened QPs if we have them*/
for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++) {
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(&endpoint->qps[qp].no_credits_pending_frags[0]);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(&endpoint->qps[qp].no_credits_pending_frags[1]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_credits_pending_frags[0]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_credits_pending_frags[1]);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
&endpoint->qps[qp].no_wqe_pending_frags[0]);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(
&endpoint->qps[qp].no_wqe_pending_frags[1]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[0]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[1]);
if(--endpoint->qps[qp].qp->users != 0)
continue;
if(endpoint->qps[qp].qp->lcl_qp != NULL)
if(ibv_destroy_qp(endpoint->qps[qp].qp->lcl_qp))
BTL_ERROR(("Failed to destroy QP:%d\n", qp));
free(endpoint->qps[qp].qp);
}
/* free the qps */
free(endpoint->qps);
endpoint->qps = NULL;
free(endpoint->rem_info.rem_qps);
free(endpoint->rem_info.rem_srqs);
/* unregister xrc recv qp */
#if HAVE_XRC
#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
if (NULL != endpoint->xrc_recv_qp) {
if(ibv_destroy_qp(endpoint->xrc_recv_qp)) {
BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp->qp_num));
} else {
BTL_VERBOSE(("Unregistered XRC Recv QP:%d\n", endpoint->xrc_recv_qp->qp_num));
}
}
#else
if (0 != endpoint->xrc_recv_qp_num) {
if(ibv_unreg_xrc_rcv_qp(endpoint->endpoint_btl->device->xrc_domain,
endpoint->xrc_recv_qp_num)) {
BTL_ERROR(("Failed to unregister XRC recv QP:%d\n", endpoint->xrc_recv_qp_num));
} else {
BTL_VERBOSE(("Unregistered XRC Recv QP:%d\n", endpoint->xrc_recv_qp_num));
}
}
#endif
#endif
OBJ_DESTRUCT(&endpoint->endpoint_lock);
/* Clean pending lists */
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(&endpoint->pending_lazy_frags);
OBJ_DESTRUCT(&endpoint->pending_lazy_frags);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(&endpoint->pending_get_frags);
OBJ_DESTRUCT(&endpoint->pending_get_frags);
MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(&endpoint->pending_put_frags);
OBJ_DESTRUCT(&endpoint->pending_put_frags);
}
static void mca_btl_wv_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
{
bool pval_clean = false;
int qp;
/* If the CPC has an endpoint_finalize function, call it */
if (NULL != endpoint->endpoint_local_cpc->cbm_endpoint_finalize) {
endpoint->endpoint_local_cpc->cbm_endpoint_finalize(endpoint);
}
/* Release CTS buffer */
ompi_btl_wv_connect_base_free_cts(endpoint);
/* Release memory resources */
do {
/* Make sure that mca_btl_wv_endpoint_connect_eager_rdma ()
* was not in "connect" or "bad" flow (failed to allocate memory)
* and changed the pointer back to NULL
*/
if(!opal_atomic_cmpset_ptr(&endpoint->eager_rdma_local.base.pval, NULL,
(void*)1)) {
if ((void*)1 != endpoint->eager_rdma_local.base.pval &&
NULL != endpoint->eager_rdma_local.base.pval) {
endpoint->endpoint_btl->super.btl_mpool->mpool_free(endpoint->endpoint_btl->super.btl_mpool,
endpoint->eager_rdma_local.base.pval,
(mca_mpool_base_registration_t*)endpoint->eager_rdma_local.reg);
pval_clean=true;
}
} else {
pval_clean=true;
}
} while (!pval_clean);
/* Close opened QPs if we have them*/
for(qp = 0; qp < mca_btl_wv_component.num_qps; qp++) {
MCA_BTL_WV_CLEAN_PENDING_FRAGS(&endpoint->qps[qp].no_credits_pending_frags[0]);
MCA_BTL_WV_CLEAN_PENDING_FRAGS(&endpoint->qps[qp].no_credits_pending_frags[1]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_credits_pending_frags[0]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_credits_pending_frags[1]);
MCA_BTL_WV_CLEAN_PENDING_FRAGS(
&endpoint->qps[qp].no_wqe_pending_frags[0]);
MCA_BTL_WV_CLEAN_PENDING_FRAGS(
&endpoint->qps[qp].no_wqe_pending_frags[1]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[0]);
OBJ_DESTRUCT(&endpoint->qps[qp].no_wqe_pending_frags[1]);
if(--endpoint->qps[qp].qp->users != 0)
continue;
if(endpoint->qps[qp].qp->lcl_qp != NULL) {
endpoint->qps[qp].qp->lcl_qp->handle->CancelOverlappedRequests();
endpoint->qps[qp].qp->lcl_qp->conn_handle->Release();
endpoint->qps[qp].qp->lcl_qp->handle->Release();
free(endpoint->qps[qp].qp->lcl_qp);
}
free(endpoint->qps[qp].qp);
}
/* free the qps */
free(endpoint->qps);
endpoint->qps = NULL;
OBJ_DESTRUCT(&endpoint->endpoint_lock);
/* Clean pending lists */
MCA_BTL_WV_CLEAN_PENDING_FRAGS(&endpoint->pending_lazy_frags);
OBJ_DESTRUCT(&endpoint->pending_lazy_frags);
MCA_BTL_WV_CLEAN_PENDING_FRAGS(&endpoint->pending_get_frags);
OBJ_DESTRUCT(&endpoint->pending_get_frags);
MCA_BTL_WV_CLEAN_PENDING_FRAGS(&endpoint->pending_put_frags);
OBJ_DESTRUCT(&endpoint->pending_put_frags);
}
