int MPID_nem_mxm_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
int str_errno = MPL_STR_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_GET_BUSINESS_CARD);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_GET_BUSINESS_CARD);
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, MXM_MPICH_ENDPOINT_KEY,
_mxm_obj.mxm_ep_addr, _mxm_obj.mxm_ep_addr_size);
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDU_bc_table_create(int rank, int size, int *nodemap, void *bc, int bc_len, int same_len,
int roots_only, void **bc_table, size_t ** bc_indices)
{
int rc, mpi_errno = MPI_SUCCESS;
int start, end, i;
char *val = NULL, *val_p;
int out_len, val_len, rem, flag;
pmix_value_t value, *pvalue;
pmix_info_t *info;
pmix_proc_t proc;
int local_rank, local_leader;
size_t my_bc_len = bc_len;
MPIR_NODEMAP_get_local_info(rank, size, nodemap, &local_size, &local_rank, &local_leader);
/* if business cards can be different length, use the max value length */
if (!same_len)
bc_len = VALLEN;
mpi_errno = MPIDU_shm_seg_alloc(bc_len * size, (void **) &segment, MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno =
MPIDU_shm_seg_commit(&memory, &barrier, local_size, local_rank, local_leader, rank,
MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (size == 1) {
memcpy(segment, bc, my_bc_len);
goto single;
}
val = MPL_malloc(VALLEN, MPL_MEM_ADDRESS);
memset(val, 0, VALLEN);
val_p = val;
rem = VALLEN;
rc = MPL_str_add_binary_arg(&val_p, &rem, "mpi", (char *) bc, my_bc_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**buscard");
MPIR_Assert(rem >= 0);
if (!roots_only || rank == local_leader) {
value.type = PMIX_STRING;
value.data.string = val;
rc = PMIx_Put(PMIX_LOCAL, "bc", &value);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_put");
rc = PMIx_Put(PMIX_REMOTE, "bc", &value);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_put");
rc = PMIx_Commit();
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_commit");
}
PMIX_INFO_CREATE(info, 1);
PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL);
rc = PMIx_Fence(&MPIR_Process.pmix_wcproc, 1, info, 1);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_fence");
PMIX_INFO_FREE(info, 1);
if (!roots_only) {
start = local_rank * (size / local_size);
end = start + (size / local_size);
if (local_rank == local_size - 1)
end += size % local_size;
for (i = start; i < end; i++) {
PMIX_PROC_CONSTRUCT(&proc);
MPL_strncpy(proc.nspace, MPIR_Process.pmix_proc.nspace, PMIX_MAX_NSLEN);
proc.rank = i;
rc = PMIx_Get(&proc, "bc", NULL, 0, &pvalue);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_get");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
PMIX_VALUE_RELEASE(pvalue);
}
} else {
int num_nodes, *node_roots;
MPIR_NODEMAP_get_node_roots(nodemap, size, &node_roots, &num_nodes);
start = local_rank * (num_nodes / local_size);
end = start + (num_nodes / local_size);
if (local_rank == local_size - 1)
end += num_nodes % local_size;
for (i = start; i < end; i++) {
PMIX_PROC_CONSTRUCT(&proc);
MPL_strncpy(proc.nspace, MPIR_Process.pmix_proc.nspace, PMIX_MAX_NSLEN);
proc.rank = i;
rc = PMIx_Get(&proc, "bc", NULL, 0, &pvalue);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmix_get");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
PMIX_VALUE_RELEASE(pvalue);
}
}
mpi_errno = MPIDU_shm_barrier(barrier, local_size);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
single:
if (!same_len) {
indices = MPL_malloc(size * sizeof(size_t), MPL_MEM_ADDRESS);
for (i = 0; i < size; i++)
indices[i] = bc_len * i;
*bc_indices = indices;
}
fn_exit:
MPL_free(val);
*bc_table = segment;
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
int str_errno = MPL_STR_SUCCESS;
int ret;
ptl_process_t my_ptl_id;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_BUSINESS_CARD);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_BUSINESS_CARD);
ret = PtlGetId(MPIDI_nem_ptl_ni, &my_ptl_id);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlgetid", "**ptlgetid %s", MPID_nem_ptl_strerror(ret));
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CHANNEL, VERBOSE, (MPL_DBG_FDEST, "Allocated NI and PT id=(%#x,%#x) pt=%#x",
my_ptl_id.phys.nid, my_ptl_id.phys.pid, MPIDI_nem_ptl_pt));
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, NID_KEY, (char *)&my_ptl_id.phys.nid, sizeof(my_ptl_id.phys.nid));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PID_KEY, (char *)&my_ptl_id.phys.pid, sizeof(my_ptl_id.phys.pid));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTI_KEY, (char *)&MPIDI_nem_ptl_pt, sizeof(MPIDI_nem_ptl_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTIG_KEY, (char *)&MPIDI_nem_ptl_get_pt,
sizeof(MPIDI_nem_ptl_get_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTIC_KEY, (char *)&MPIDI_nem_ptl_control_pt,
sizeof(MPIDI_nem_ptl_control_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTIR_KEY, (char *)&MPIDI_nem_ptl_rpt_pt,
sizeof(MPIDI_nem_ptl_rpt_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTIRG_KEY, (char *)&MPIDI_nem_ptl_get_rpt_pt,
sizeof(MPIDI_nem_ptl_get_rpt_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPL_str_add_binary_arg(bc_val_p, val_max_sz_p, PTIRC_KEY, (char *)&MPIDI_nem_ptl_control_rpt_pt,
sizeof(MPIDI_nem_ptl_control_rpt_pt));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPL_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDU_bc_table_create(int rank, int size, int *nodemap, void *bc, int bc_len, int same_len,
int roots_only, void **bc_table, size_t ** bc_indices)
{
int rc, mpi_errno = MPI_SUCCESS;
int start, end, i;
int key_max, val_max, name_max, out_len, rem;
char *kvsname = NULL, *key = NULL, *val = NULL, *val_p;
int local_rank = -1, local_leader = -1;
size_t my_bc_len = bc_len;
MPIR_NODEMAP_get_local_info(rank, size, nodemap, &local_size, &local_rank, &local_leader);
rc = PMI_KVS_Get_name_length_max(&name_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_name_length_max");
rc = PMI_KVS_Get_key_length_max(&key_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_key_length_max");
rc = PMI_KVS_Get_value_length_max(&val_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_value_length_max");
/* if business cards can be different length, use the max value length */
if (!same_len)
bc_len = val_max;
mpi_errno = MPIDU_shm_seg_alloc(bc_len * size, (void **) &segment, MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno =
MPIDU_shm_seg_commit(&memory, &barrier, local_size, local_rank, local_leader, rank,
MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (size == 1) {
memcpy(segment, bc, my_bc_len);
goto single;
}
kvsname = MPL_malloc(name_max, MPL_MEM_ADDRESS);
MPIR_Assert(kvsname);
rc = PMI_KVS_Get_my_name(kvsname, name_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_my_name");
val = MPL_malloc(val_max, MPL_MEM_ADDRESS);
memset(val, 0, val_max);
val_p = val;
rem = val_max;
rc = MPL_str_add_binary_arg(&val_p, &rem, "mpi", (char *) bc, my_bc_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**buscard");
MPIR_Assert(rem >= 0);
key = MPL_malloc(key_max, MPL_MEM_ADDRESS);
MPIR_Assert(key);
if (!roots_only || rank == local_leader) {
sprintf(key, "bc-%d", rank);
rc = PMI_KVS_Put(kvsname, key, val);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_put");
rc = PMI_KVS_Commit(kvsname);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_commit");
}
rc = PMI_Barrier();
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_barrier");
if (!roots_only) {
start = local_rank * (size / local_size);
end = start + (size / local_size);
if (local_rank == local_size - 1)
end += size % local_size;
for (i = start; i < end; i++) {
sprintf(key, "bc-%d", i);
rc = PMI_KVS_Get(kvsname, key, val, val_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
}
} else {
int num_nodes, *node_roots;
MPIR_NODEMAP_get_node_roots(nodemap, size, &node_roots, &num_nodes);
start = local_rank * (num_nodes / local_size);
end = start + (num_nodes / local_size);
if (local_rank == local_size - 1)
end += num_nodes % local_size;
for (i = start; i < end; i++) {
sprintf(key, "bc-%d", node_roots[i]);
rc = PMI_KVS_Get(kvsname, key, val, val_max);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
}
MPL_free(node_roots);
}
mpi_errno = MPIDU_shm_barrier(barrier, local_size);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
single:
if (!same_len) {
indices = MPL_malloc(size * sizeof(size_t), MPL_MEM_ADDRESS);
MPIR_Assert(indices);
for (i = 0; i < size; i++)
indices[i] = bc_len * i;
*bc_indices = indices;
}
fn_exit:
MPL_free(kvsname);
MPL_free(key);
MPL_free(val);
*bc_table = segment;
return mpi_errno;
fn_fail:
goto fn_exit;
}
