sshmem_mkey_t *mca_spml_ikrit_register(void* addr,
size_t size,
uint64_t shmid,
int *count)
{
int i;
sshmem_mkey_t *mkeys;
mxm_error_t err;
mxm_mem_key_t *m_key;
int my_rank = oshmem_my_proc_id();
*count = 0;
mkeys = (sshmem_mkey_t *) calloc(1, MXM_PTL_LAST * sizeof(*mkeys));
if (!mkeys) {
return NULL ;
}
for (i = 0; i < MXM_PTL_LAST; i++) {
mkeys[i].u.key = MAP_SEGMENT_SHM_INVALID;
switch (i) {
case MXM_PTL_SHM:
if ((int)shmid != MAP_SEGMENT_SHM_INVALID) {
mkeys[i].u.key = shmid;
mkeys[i].va_base = 0;
} else {
mkeys[i].len = 0;
mkeys[i].va_base = addr;
}
mkeys[i].spml_context = 0;
break;
case MXM_PTL_RDMA:
mkeys[i].va_base = addr;
mkeys[i].spml_context = 0;
if (mca_spml_ikrit.ud_only) {
mkeys[i].len = 0;
break;
}
err = mxm_mem_map(mca_spml_ikrit.mxm_context, &addr, &size, 0, 0, 0);
if (MXM_OK != err) {
SPML_ERROR("Failed to register memory: %s", mxm_error_string(err));
goto error_out;
}
mkeys[i].spml_context = (void *)(unsigned long)size;
m_key = malloc(sizeof(*m_key));
if (NULL == m_key) {
SPML_ERROR("Failed to allocate m_key memory");
goto error_out;
}
mkeys[i].len = sizeof(*m_key);
mkeys[i].u.data = m_key;
err = mxm_mem_get_key(mca_spml_ikrit.mxm_context, addr, m_key);
if (MXM_OK != err) {
SPML_ERROR("Failed to get memory key: %s", mxm_error_string(err));
goto error_out;
}
break;
default:
SPML_ERROR("unsupported PTL: %d", i);
goto error_out;
}
SPML_VERBOSE(5,
"rank %d ptl %d addr %p size %llu %s",
my_rank, i, addr, (unsigned long long)size,
mca_spml_base_mkey2str(&mkeys[i]));
mca_spml_ikrit_cache_mkeys(&mkeys[i], memheap_find_segnum(addr), my_rank, i);
}
*count = MXM_PTL_LAST;
return mkeys;
error_out:
mca_spml_ikrit_deregister(mkeys);
return NULL;
}
sshmem_mkey_t *mca_spml_ikrit_register(void* addr,
size_t size,
uint64_t shmid,
int *count)
{
int i;
sshmem_mkey_t *mkeys;
#if MXM_API >= MXM_VERSION(2,0)
mxm_error_t err;
mxm_mem_key_t *m_key;
#endif
*count = 0;
mkeys = (sshmem_mkey_t *) calloc(1, MXM_PTL_LAST * sizeof(*mkeys));
if (!mkeys) {
return NULL ;
}
for (i = 0; i < MXM_PTL_LAST; i++) {
mkeys[i].u.key = MAP_SEGMENT_SHM_INVALID;
switch (i) {
case MXM_PTL_SHM:
if ((int)shmid != MAP_SEGMENT_SHM_INVALID) {
mkeys[i].u.key = shmid;
mkeys[i].va_base = 0;
} else {
mkeys[i].len = 0;
mkeys[i].va_base = addr;
}
mkeys[i].spml_context = 0;
break;
#if MXM_API < MXM_VERSION(2,0)
case MXM_PTL_SELF:
mkeys[i].len = 0;
mkeys[i].spml_context = 0;
mkeys[i].va_base = addr;
break;
#endif
case MXM_PTL_RDMA:
mkeys[i].va_base = addr;
mkeys[i].spml_context = 0;
#if MXM_API < MXM_VERSION(2,0)
mkeys[i].len = 0;
#else
if (mca_spml_ikrit.ud_only && !mca_spml_ikrit.hw_rdma_channel) {
mkeys[i].len = 0;
break;
}
err = mxm_mem_map(mca_spml_ikrit.mxm_context, &addr, &size, 0, 0, 0);
if (MXM_OK != err) {
SPML_ERROR("Failed to register memory: %s", mxm_error_string(err));
goto error_out;
}
mkeys[i].spml_context = (void *)(unsigned long)size;
m_key = malloc(sizeof(*m_key));
if (NULL == m_key) {
SPML_ERROR("Failed to allocate m_key memory");
goto error_out;
}
mkeys[i].len = sizeof(*m_key);
mkeys[i].u.data = m_key;
err = mxm_mem_get_key(mca_spml_ikrit.mxm_context, addr, m_key);
if (MXM_OK != err) {
SPML_ERROR("Failed to get memory key: %s", mxm_error_string(err));
goto error_out;
}
#endif
break;
default:
SPML_ERROR("unsupported PTL: %d", i);
goto error_out;
}
SPML_VERBOSE(5,
"rank %d ptl %d addr %p size %llu %s",
oshmem_proc_local_proc->super.proc_name.vpid, i, addr, (unsigned long long)size,
mca_spml_base_mkey2str(&mkeys[i]));
}
*count = MXM_PTL_LAST;
return mkeys;
error_out:
mca_spml_ikrit_deregister(mkeys);
return NULL;
}
