void*
shmem_internal_shmalloc(size_t size)
{
void *ret;
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
ret = dlmalloc(size);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
return ret;
}
void
shmem_free(void *ptr)
{
SHMEM_ERR_CHECK_INITIALIZED();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
dlfree(ptr);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
shmem_internal_barrier_all();
}
void
FC_SHPDEALLOC(void **addr, fortran_integer_t *errcode, fortran_integer_t *want_abort)
{
SHMEM_ERR_CHECK_INITIALIZED();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
dlfree(*addr);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
*errcode = 0;
shmem_internal_barrier_all();
}
void mca_spml_ucx_ctx_destroy(shmem_ctx_t ctx)
{
MCA_SPML_CALL(quiet(ctx));
SHMEM_MUTEX_LOCK(mca_spml_ucx.internal_mutex);
_ctx_remove(&mca_spml_ucx.active_array, (mca_spml_ucx_ctx_t *)ctx);
_ctx_add(&mca_spml_ucx.idle_array, (mca_spml_ucx_ctx_t *)ctx);
SHMEM_MUTEX_UNLOCK(mca_spml_ucx.internal_mutex);
if (!mca_spml_ucx.active_array.ctxs_count) {
opal_progress_unregister(spml_ucx_ctx_progress);
}
}
void
shmem_free(void *ptr)
{
SHMEM_ERR_CHECK_INITIALIZED();
if (ptr != NULL) {
SHMEM_ERR_CHECK_SYMMETRIC_HEAP(ptr);
}
shmem_internal_barrier_all();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
dlfree(ptr);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
}
void *
shmem_align(size_t alignment, size_t size)
{
void *ret;
SHMEM_ERR_CHECK_INITIALIZED();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
ret = dlmemalign(alignment, size);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
shmem_internal_barrier_all();
return ret;
}
void *
shmem_realloc(void *ptr, size_t size)
{
void *ret;
SHMEM_ERR_CHECK_INITIALIZED();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
ret = dlrealloc(ptr, size);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
shmem_internal_barrier_all();
return ret;
}
void
FC_SHPCLMOVE(void **addr, fortran_integer_t *length, fortran_integer_t *errcode, fortran_integer_t *want_abort)
{
void *ret;
SHMEM_ERR_CHECK_INITIALIZED();
SHMEM_ERR_CHECK_SYMMETRIC_HEAP(*addr);
if (*length <= 0) {
if (0 == *want_abort) {
fprintf(stderr, "[%03d] shpclmove failure (invalid length)\n",
shmem_internal_my_pe);
RAISE_ERROR(-1);
} else {
*errcode = -1;
return;
}
}
shmem_internal_barrier_all();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
ret = dlrealloc(*addr, *length * 4); /* length is number of 32 bit words */
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
if (*addr != NULL) {
if (*addr == ret) {
*errcode = 0;
} else {
*errcode = 1;
}
*addr = ret;
} else {
if (0 == *want_abort) {
fprintf(stderr, "[%03d] shpclmove failure\n",
shmem_internal_my_pe);
RAISE_ERROR(-2);
} else {
*errcode = -2;
return;
}
}
shmem_internal_barrier_all();
}
void *
shmem_realloc(void *ptr, size_t size)
{
void *ret;
SHMEM_ERR_CHECK_INITIALIZED();
if (ptr != NULL) {
SHMEM_ERR_CHECK_SYMMETRIC_HEAP(ptr);
}
shmem_internal_barrier_all();
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
ret = dlrealloc(ptr, size);
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
shmem_internal_barrier_all();
return ret;
}
void
FC_SHPALLOC(void **addr, fortran_integer_t *length, fortran_integer_t *errcode, fortran_integer_t *want_abort)
{
size_t len;
SHMEM_ERR_CHECK_INITIALIZED();
len = ((size_t) *length) * 4;
*errcode = 0;
if (len == 0) {
if (0 == *want_abort) {
*errcode = -1;
return;
} else {
fprintf(stderr, "[%03d] ERROR: shpalloc failure (invalid length). Aborting job.\n",
shmem_internal_my_pe);
RAISE_ERROR(1);
}
}
SHMEM_MUTEX_LOCK(shmem_internal_mutex_alloc);
*addr = dlmalloc(len); /* length is number of 32 bit words */
SHMEM_MUTEX_UNLOCK(shmem_internal_mutex_alloc);
if (*addr == NULL) {
if (0 == *want_abort) {
*errcode = -2;
return;
} else {
fprintf(stderr, "[%03d] ERROR: shpalloc failure. Aborting job.\n",
shmem_internal_my_pe);
RAISE_ERROR(1);
}
}
shmem_internal_barrier_all();
}
int mca_spml_ucx_ctx_create(long options, shmem_ctx_t *ctx)
{
mca_spml_ucx_ctx_t *ucx_ctx;
ucp_worker_params_t params;
ucp_ep_params_t ep_params;
size_t i, j, nprocs = oshmem_num_procs();
ucs_status_t err;
int my_pe = oshmem_my_proc_id();
size_t len;
spml_ucx_mkey_t *ucx_mkey;
sshmem_mkey_t *mkey;
int rc = OSHMEM_ERROR;
ucx_ctx = malloc(sizeof(mca_spml_ucx_ctx_t));
ucx_ctx->options = options;
params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || options & SHMEM_CTX_PRIVATE || options & SHMEM_CTX_SERIALIZED) {
params.thread_mode = UCS_THREAD_MODE_SINGLE;
} else {
params.thread_mode = UCS_THREAD_MODE_MULTI;
}
err = ucp_worker_create(mca_spml_ucx.ucp_context, ¶ms,
&ucx_ctx->ucp_worker);
if (UCS_OK != err) {
free(ucx_ctx);
return OSHMEM_ERROR;
}
ucx_ctx->ucp_peers = (ucp_peer_t *) calloc(nprocs, sizeof(*(ucx_ctx->ucp_peers)));
if (NULL == ucx_ctx->ucp_peers) {
goto error;
}
if (mca_spml_ucx.active_array.ctxs_count == 0) {
opal_progress_register(spml_ucx_ctx_progress);
}
for (i = 0; i < nprocs; i++) {
ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS;
ep_params.address = (ucp_address_t *)(mca_spml_ucx.remote_addrs_tbl[i]);
err = ucp_ep_create(ucx_ctx->ucp_worker, &ep_params,
&ucx_ctx->ucp_peers[i].ucp_conn);
if (UCS_OK != err) {
SPML_ERROR("ucp_ep_create(proc=%d/%d) failed: %s", i, nprocs,
ucs_status_string(err));
goto error2;
}
for (j = 0; j < MCA_MEMHEAP_SEG_COUNT; j++) {
mkey = &memheap_map->mem_segs[j].mkeys_cache[i][0];
ucx_mkey = &ucx_ctx->ucp_peers[i].mkeys[j].key;
err = ucp_ep_rkey_unpack(ucx_ctx->ucp_peers[i].ucp_conn,
mkey->u.data,
&ucx_mkey->rkey);
if (UCS_OK != err) {
SPML_UCX_ERROR("failed to unpack rkey");
goto error2;
}
mca_spml_ucx_cache_mkey(ucx_ctx, mkey, j, i);
}
}
SHMEM_MUTEX_LOCK(mca_spml_ucx.internal_mutex);
_ctx_add(&mca_spml_ucx.active_array, ucx_ctx);
SHMEM_MUTEX_UNLOCK(mca_spml_ucx.internal_mutex);
(*ctx) = (shmem_ctx_t)ucx_ctx;
return OSHMEM_SUCCESS;
error2:
for (i = 0; i < nprocs; i++) {
if (ucx_ctx->ucp_peers[i].ucp_conn) {
ucp_ep_destroy(ucx_ctx->ucp_peers[i].ucp_conn);
}
}
if (ucx_ctx->ucp_peers)
free(ucx_ctx->ucp_peers);
error:
ucp_worker_destroy(ucx_ctx->ucp_worker);
free(ucx_ctx);
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
SPML_ERROR("ctx create FAILED rc=%d", rc);
return rc;
}
