static int test_item6(void)
{
int rc = 0;
char val[PMI2_MAX_VALLEN];
int len;
const char *tkey = __FUNCTION__;
const char *tval = __FILE__;
if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
log_fatal("PMI2_KVS_Put %d\n", rc);
return rc;
}
if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(NULL, PMI2_ID_NULL, tkey, val, sizeof(val), &len))) {
log_fatal("PMI2_KVS_Get %d\n", rc);
return rc;
}
log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
log_assert((int)strlen(tval) == len, "value does not meet expectation");
log_assert(!strcmp(tval, val), "value does not meet expectation");
return rc;
}
void pmi_put(char *key, void *value, size_t valuelen, int rank) {
int res = snprintf(kvs_key, max_key_len, "ofi-%lu-%s", (long unsigned) rank, key);
assert(res > 0);
res = encode(value, valuelen, kvs_value, max_val_len);
assert(res == 0);
res = PMI2_KVS_Put(kvs_key, kvs_value);
assert(res == PMI2_SUCCESS);
}
/* Because Cray uses PMI2 extensions for some, but not all,
* PMI functions, we define a set of wrappers for those
* common functions we will use
*/
static int kvs_put(const char *key, const char *value)
{
#if WANT_CRAY_PMI2_EXT
return PMI2_KVS_Put(key, value);
#else
return PMI_KVS_Put(pmi_kvs_name, key, value);
#endif
}
void pmi_put_key_rem(char *key, int *key_val, int key_size)
{
char *encoded = pmi_encode(key_val, key_size * sizeof(int));
if( NULL == encoded ){
fprintf(stderr, "pmi_encode: error on key: %s\n", key);
abort();
}
PMI2_KVS_Put(key, encoded);
}
static int kvs_put(const char key[], const char value[])
{
int rc;
rc = PMI2_KVS_Put(key, value);
if( PMI2_SUCCESS != rc ){
OPAL_PMI_ERROR(rc, "PMI2_KVS_Put");
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
void do_kvs_put(void *value, size_t sz) {
int rc;
do_encode(value, sz);
#if USE_PMI2_API
rc = PMI2_KVS_Put(kvs_key, kvs_value);
gasneti_assert(PMI2_SUCCESS == rc);
#else
rc = PMI_KVS_Put(kvs_name, kvs_key, kvs_value);
gasneti_assert(PMI_SUCCESS == rc);
#endif
}
int
shmem_runtime_put(char *key, void *value, size_t valuelen)
{
snprintf(kvs_key, max_key_len, "shmem-%lu-%s", (long unsigned) rank, key);
if (0 != encode(value, valuelen, kvs_value, max_val_len)) {
return 1;
}
if (PMI2_SUCCESS != PMI2_KVS_Put(kvs_key, kvs_value)) {
return 2;
}
return 0;
}
static int kvs_put(const char key[], const char value[])
{
int rc;
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray kvs_put key %s value %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), key, value);
rc = PMI2_KVS_Put(key, value);
if( PMI_SUCCESS != rc ){
OPAL_PMI_ERROR(rc, "PMI2_KVS_Put");
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
int mca_common_pmi_put(const char *kvs_name,
const char *key, const char *value)
{
int rc;
#if WANT_PMI2_SUPPORT
if( mca_common_pmi_version == 2 ){
if( PMI2_SUCCESS != PMI2_KVS_Put(key, value) ){
// FIXME: OPAL_PMI2_ERROR(rc, "PMI2_KVS_Put");
return OPAL_ERROR;
}
}
else
#endif
{
rc = PMI_KVS_Put(kvs_name, key, value);
if( PMI_SUCCESS != rc ){
OPAL_PMI_ERROR(rc, "PMI_KVS_Put");
return OPAL_ERROR;
}
}
return OPAL_SUCCESS;
}
static int test_item8(void)
{
int rc = 0;
int len;
char tkey[PMI2_MAX_VALLEN];
char tval[PMI2_MAX_VALLEN];
char val[PMI2_MAX_VALLEN];
int i = 0;
for (i = 0; i < size; i++) {
sprintf(tkey, "KEY-%d", i);
sprintf(tval, "VALUE-%d", i);
if (i == rank) {
if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
log_fatal("PMI2_KVS_Put [%s=%s] %d\n", tkey, tval, rc);
return rc;
}
}
if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
log_fatal("PMI2_KVS_Fence %d\n", rc);
return rc;
}
if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(jobid, PMI2_ID_NULL, tkey, val, sizeof(val), &len))) {
log_fatal("PMI2_KVS_Get [%s=?] %d\n", tkey, rc);
return rc;
}
log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
log_assert((int)strlen(tval) == len, "value does not meet expectation");
log_assert(!strcmp(tval, val), "value does not meet expectation");
}
return rc;
}
int MPIDI_PG_SetConnInfo( int rank, const char *connString )
{
#ifdef USE_PMI2_API
int mpi_errno = MPI_SUCCESS;
int len;
char key[PMI2_MAX_KEYLEN];
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
len = MPL_snprintf(key, sizeof(key), "P%d-businesscard", rank);
MPIR_ERR_CHKANDJUMP1(len < 0 || len > sizeof(key), mpi_errno, MPI_ERR_OTHER, "**snprintf", "**snprintf %d", len);
mpi_errno = PMI2_KVS_Put(key, connString);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = PMI2_KVS_Fence();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#else
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int len;
char key[128];
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_Assert(pg_world->connData);
len = MPL_snprintf(key, sizeof(key), "P%d-businesscard", rank);
if (len < 0 || len > sizeof(key)) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**snprintf",
"**snprintf %d", len);
}
pmi_errno = PMI_KVS_Put(pg_world->connData, key, connString );
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_put",
"**pmi_kvs_put %d", pmi_errno);
}
pmi_errno = PMI_KVS_Commit(pg_world->connData);
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_commit",
"**pmi_kvs_commit %d", pmi_errno);
}
pmi_errno = PMI_Barrier();
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_barrier",
"**pmi_barrier %d", pmi_errno);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#endif
}
static int test_item9(void)
{
int rc = 0;
int i, j, r;
char symb, symb_start = 'a';
int fence_cnt;
int fence_num = random_value(2, 10);
int keys_per_fence = random_value(10, 100);
int val_size = random_value(10, PMI2_MAX_VALLEN / 10);
int keys_total = 0;
fence_cnt = 0;
while (fence_cnt < fence_num) {
log_info("fence_cnt=%d of fence_num=%d keys_per_fence=%d keys_total=%d val_size=%d\n",
fence_cnt, fence_num, keys_per_fence, keys_total, val_size);
symb = symb_start;
for (i = 0; i < keys_per_fence; i++) {
char key[PMI2_MAX_KEYLEN];
char val[PMI2_MAX_VALLEN] = "";
sprintf(key, "RANK%d-key-%d", rank, i + keys_total);
for (j = 0; j < val_size; j++) {
val[j] = symb;
}
symb++;
if (symb > 'z') {
symb = 'a';
}
if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(key, val))) {
log_fatal("PMI2_KVS_Put [%s=%s] %d\n", key, val, rc);
return rc;
}
log_info("PMI2_KVS_Put [rank=%d %s] %d\n", rank, key, rc);
}
symb_start = symb;
keys_total += keys_per_fence;
if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
log_fatal("PMI2_KVS_Fence %d\n", rc);
return rc;
}
for (r = 0; r < size; r++) {
int len;
symb = 'a';
for (i = 0; i < keys_total; i++) {
char key[PMI2_MAX_KEYLEN];
char val[PMI2_MAX_VALLEN] = "";
sprintf(key, "RANK%d-key-%d", r, i);
if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(jobid, r, key, val, sizeof(val), &len))) {
log_fatal("PMI2_KVS_Get [%s=?] %d\n", key, rc);
return rc;
}
log_info("PMI2_KVS_Get [rank=%d %s] %d\n", rank, key, rc);
if (len != val_size) {
log_fatal("%d: failure on rank %d, key #%d: len mismatch:"
" %d instead of %d\n", rank, r, i, len, val_size);
}
for (j = 0; j < val_size; j++) {
if (val[j] != symb) {
log_fatal("%d: failure on rank %d, key #%d: value mismatch"
" at symb %d: \'%c\' instead of \'%c\'\n", rank,
r, i, j, val[j], symb);
}
}
symb++;
if (symb > 'z') {
symb = 'a';
}
}
}
fence_cnt++;
}
return rc;
}
int
main(int argc, char **argv)
{
int rank;
int size;
int appnum;
int spawned;
int flag;
int len;
int i;
struct timeval tv;
struct timeval tv2;
char jobid[128];
char key[128];
char val[128];
char buf[128];
{
int x = 1;
while (x == 0) {
sleep(2);
}
}
gettimeofday(&tv, NULL);
srand(tv.tv_sec);
PMI2_Init(&spawned, &size, &rank, &appnum);
PMI2_Job_GetId(jobid, sizeof(buf));
memset(val, 0, sizeof(val));
PMI2_Info_GetJobAttr("mpi_reserved_ports",
val,
PMI2_MAX_ATTRVALUE,
&flag);
sprintf(key, "mpi_reserved_ports");
PMI2_KVS_Put(key, val);
memset(val, 0, sizeof(val));
sprintf(buf, "PMI_netinfo_of_task");
PMI2_Info_GetJobAttr(buf,
val,
PMI2_MAX_ATTRVALUE,
&flag);
sprintf(key, buf);
PMI2_KVS_Put(key, val);
memset(val, 0, sizeof(val));
sprintf(key, "david@%d", rank);
sprintf(val, "%s", mrand(97, 122));
PMI2_KVS_Put(key, val);
PMI2_KVS_Fence();
for (i = 0; i < size; i++) {
memset(val, 0, sizeof(val));
sprintf(key, "PMI_netinfo_of_task");
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
memset(val, 0, sizeof(val));
sprintf(key, "david@%d", rank);
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
memset(val, 0, sizeof(val));
sprintf(key, "mpi_reserved_ports");
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
}
PMI2_Finalize();
gettimeofday(&tv2, NULL);
printf("%f\n",
((tv2.tv_sec - tv.tv_sec) * 1000.0
+ (tv2.tv_usec - tv.tv_usec) / 1000.0));
return 0;
}
void pmi_put_double(char *key, double val)
{
char buf[128];
sprintf(buf, "%lf", val);
PMI2_KVS_Put(key, buf);
}
int MPIDU_bc_table_create(int rank, int size, int *nodemap, void *bc, int bc_len, int same_len,
int roots_only, void **bc_table, size_t ** bc_indices)
{
int rc, mpi_errno = MPI_SUCCESS;
int start, end, i;
int out_len, val_len, rem;
char *key = NULL, *val = NULL, *val_p;
int local_rank, local_leader;
size_t my_bc_len = bc_len;
MPIR_NODEMAP_get_local_info(rank, size, nodemap, &local_size, &local_rank, &local_leader);
/* if business cards can be different length, use the max value length */
if (!same_len)
bc_len = PMI2_MAX_VALLEN;
mpi_errno = MPIDU_shm_seg_alloc(bc_len * size, (void **) &segment, MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno =
MPIDU_shm_seg_commit(&memory, &barrier, local_size, local_rank, local_leader, rank,
MPL_MEM_ADDRESS);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (size == 1) {
memcpy(segment, bc, my_bc_len);
goto single;
}
val = MPL_malloc(PMI2_MAX_VALLEN, MPL_MEM_ADDRESS);
memset(val, 0, PMI2_MAX_VALLEN);
val_p = val;
rem = PMI2_MAX_VALLEN;
rc = MPL_str_add_binary_arg(&val_p, &rem, "mpi", (char *) bc, my_bc_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**buscard");
MPIR_Assert(rem >= 0);
key = MPL_malloc(PMI2_MAX_KEYLEN, MPL_MEM_ADDRESS);
MPIR_Assert(key);
if (!roots_only || rank == local_leader) {
sprintf(key, "bc-%d", rank);
rc = PMI2_KVS_Put(key, val);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvsput");
}
rc = PMI2_KVS_Fence();
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvsfence");
if (!roots_only) {
start = local_rank * (size / local_size);
end = start + (size / local_size);
if (local_rank == local_size - 1)
end += size % local_size;
for (i = start; i < end; i++) {
sprintf(key, "bc-%d", i);
rc = PMI2_KVS_Get(NULL, -1, key, val, PMI2_MAX_VALLEN, &val_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvsget");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
}
} else {
int num_nodes, *node_roots;
MPIR_NODEMAP_get_node_roots(nodemap, size, &node_roots, &num_nodes);
start = local_rank * (num_nodes / local_size);
end = start + (num_nodes / local_size);
if (local_rank == local_size - 1)
end += num_nodes % local_size;
for (i = start; i < end; i++) {
sprintf(key, "bc-%d", node_roots[i]);
rc = PMI2_KVS_Get(NULL, -1, key, val, PMI2_MAX_VALLEN, &val_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**pmi_kvsget");
rc = MPL_str_get_binary_arg(val, "mpi", &segment[i * bc_len], bc_len, &out_len);
MPIR_ERR_CHKANDJUMP(rc, mpi_errno, MPI_ERR_OTHER, "**argstr_missinghost");
}
MPL_free(node_roots);
}
mpi_errno = MPIDU_shm_barrier(barrier, local_size);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
single:
if (!same_len) {
indices = MPL_malloc(size * sizeof(size_t), MPL_MEM_ADDRESS);
for (i = 0; i < size; i++)
indices[i] = bc_len * i;
*bc_indices = indices;
}
fn_exit:
MPL_free(key);
MPL_free(val);
*bc_table = segment;
return mpi_errno;
fn_fail:
goto fn_exit;
}
