int chpl_env_str_to_int_pct(const char* evName, const char* evVal,
int dflt, chpl_bool doWarn) {
int val;
int evIdx;
if (evVal == NULL)
return dflt;
if (sscanf(evVal, "%d%n", &val, &evIdx) == 1
&& val > 0
&& evVal[evIdx] == '%') {
return val;
}
if (doWarn) {
if (evName == NULL) {
chpl_msg(1,
"warning: env var improper int percentage \"%s\", assuming "
"%d\n",
evVal, dflt);
} else {
chpl_msg(1,
"warning: CHPL_RT_%s improper int percentage \"%s\", assuming "
"%d\n",
evName, evVal, dflt);
}
}
return dflt;
}
static void handleDeprecatedConfig(const char* varName,
const char* value,
const char* envVarName) {
if (getenv(envVarName) == NULL) {
chpl_msg(0,
"warning: The config variable \"%s\" is deprecated. Please use\n"
" the environment variable \"%s\" instead.\n",
varName, envVarName);
if (value != NULL)
setenv(envVarName, value, 0);
}
else
chpl_msg(0,
"warning: The config variable \"%s\" is deprecated, and is\n"
" overridden by the environment variable \"%s\".\n",
varName, envVarName);
}
//
// a final comm layer stub before barrier synching and calling into
// the user code. It is recommended that a debugging message be
// printed here indicating that each locale has started using
// chpl_msg() and a verbosity level of 2 (which will cause it to be
// displayed using the -v flag).
//
// Cannot call spr_unify() here because there is a barrier afterwards.
void chpl_comm_rollcall(void)
{
qthread_debug(CHAPEL_CALLS, "[%d] begin\n", chpl_localeID);
chpl_msg(2, "executing on locale %d of %d locale(s): %s\n", chpl_localeID,
chpl_numLocales, chpl_localeName());
qthread_debug(CHAPEL_CALLS, "[%d] end\n", chpl_localeID);
}
size_t chpl_env_str_to_size(const char* evName, const char* evVal,
size_t dflt) {
chpl_bool okay;
int scnCnt;
size_t val;
char units;
if (evVal == NULL)
return dflt;
//
// Try to collect a strictly unsigned decimal, octal, or hexadecimal
// number, with an optional following units character.
//
okay = false;
if (isdigit(evVal[0])
&& (scnCnt = sscanf(evVal, "%zi%c", &val, &units)) > 0) {
okay = true;
if (scnCnt == 2 && strchr("kKmMgG", units) != NULL) {
switch (units) {
case 'k' : case 'K': val <<= 10; break;
case 'm' : case 'M': val <<= 20; break;
case 'g' : case 'G': val <<= 30; break;
}
}
}
if (!okay) {
if (evName == NULL) {
chpl_msg(1,
"warning: env var improper size value \"%s\", assuming %zd\n",
evVal, dflt);
} else {
chpl_msg(1,
"warning: CHPL_RT_%s improper size value \"%s\", assuming "
"%zd\n",
evName, evVal, dflt);
}
return dflt;
}
return val;
}
chpl_bool chpl_env_str_to_bool(const char* evName, const char* evVal,
chpl_bool dflt) {
if (evVal == NULL)
return dflt;
if (strchr("0fFnN", evVal[0]) != NULL)
return false;
if (strchr("1tTyY", evVal[0]) != NULL)
return true;
if (evName == NULL) {
chpl_msg(1,
"warning: env var improper bool value \"%s\", assuming %c\n",
evVal, (dflt ? 'T' : 'F'));
} else {
chpl_msg(1,
"warning: CHPL_RT_%s improper bool value \"%s\", assuming %c\n",
evName, evVal, (dflt ? 'T' : 'F'));
}
return dflt;
}
int64_t chpl_env_str_to_int(const char* evName, const char* evVal,
int64_t dflt) {
int64_t val;
if (evVal == NULL)
return dflt;
if (sscanf(evVal, "%" SCNi64, &val) == 1)
return val;
if (evName == NULL) {
chpl_msg(1,
"warning: env var improper int value \"%s\", assuming "
"%" PRId64 "\n",
evVal, dflt);
} else {
chpl_msg(1,
"warning: CHPL_RT_%s improper int value \"%s\", assuming "
"%" PRId64 "\n",
evName, evVal, dflt);
}
return dflt;
}
void chpl_comm_barrier(const char *msg) {
chpl_msg(2, "%d: enter barrier for '%s'\n", chpl_nodeID, msg);
if (chpl_numNodes == 1) {
return;
}
if (!progress_threads_running()) {
// Comm layer setup is not complete yet; use OOB barrier
chpl_comm_ofi_oob_barrier();
} else {
// Use OOB barrier for now, but we can do better in the future
chpl_comm_ofi_oob_barrier();
}
}
void chpl_comm_rollcall(void) {
chpl_msg(2, "executing on a single node\n");
}
void chpl_comm_rollcall(void) {
// Do this again to clear out any comms that happened during initialization
chpl_resetCommDiagnosticsHere();
chpl_msg(2, "executing on node %d of %d node(s): %s\n", chpl_nodeID,
chpl_numNodes, chpl_nodeName());
}
static void libfabric_init() {
int i;
struct fi_info *info = NULL;
struct fi_info *hints = fi_allocinfo();
struct fi_av_attr av_attr = {0};
struct fi_cq_attr cq_attr = {0};
int max_tx_ctx, max_rx_ctx;
int comm_concurrency;
int rx_ctx_cnt;
int rx_ctx_bits = 0;
hints->mode = ~0;
hints->caps = FI_RMA
| FI_ATOMIC
| FI_SOURCE /* do we want this? */
| FI_READ
| FI_WRITE
| FI_REMOTE_READ
| FI_REMOTE_WRITE
| FI_MULTI_RECV
| FI_FENCE;
hints->addr_format = FI_FORMAT_UNSPEC;
#if defined(CHPL_COMM_SUBSTRATE_SOCKETS)
//
// fi_freeinfo(hints) will free() hints->fabric_attr->prov_name; this
// is documented, though poorly. So, get that space from malloc().
//
{
const char s[] = "sockets";
char* sDup = sys_malloc(sizeof(s));
strcpy(sDup, s);
hints->fabric_attr->prov_name = sDup;
}
#elif defined(CHPL_COMM_SUBSTRATE_GNI)
#error "Substrate GNI not supported"
#else
#error "Substrate type not supported"
#endif
/* connectionless reliable */
hints->ep_attr->type = FI_EP_RDM;
hints->domain_attr->threading = FI_THREAD_UNSPEC;
hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
hints->domain_attr->av_type = FI_AV_TABLE;
hints->domain_attr->mr_mode = FI_MR_SCALABLE;
hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
// hints->domain_attr->cq_data_size
hints->tx_attr->op_flags = FI_COMPLETION;
hints->rx_attr->op_flags = FI_COMPLETION;
OFICHKERR(fi_getinfo(FI_VERSION(1,0), NULL, NULL, 0, hints, &info));
if (info == NULL) {
chpl_internal_error("No fabrics detected.");
} else {
#ifdef PRINT_FI_GETINFO
struct fi_info *cur;
for (cur = info; cur; cur = cur->next) {
printf("---\n");
printf("%s", fi_tostr(cur, FI_TYPE_INFO));
}
printf("\n");
#endif
}
ofi.num_am_ctx = 1; // Would we ever want more?
max_tx_ctx = info->domain_attr->max_ep_tx_ctx;
max_rx_ctx = info->domain_attr->max_ep_rx_ctx;
comm_concurrency = get_comm_concurrency();
ofi.num_tx_ctx = comm_concurrency+ofi.num_am_ctx > max_tx_ctx ?
max_tx_ctx-ofi.num_am_ctx : comm_concurrency;
ofi.num_rx_ctx = comm_concurrency+ofi.num_am_ctx > max_rx_ctx ?
max_rx_ctx-ofi.num_am_ctx : comm_concurrency;
info->ep_attr->tx_ctx_cnt = ofi.num_tx_ctx + ofi.num_am_ctx;
info->ep_attr->rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;
OFICHKERR(fi_fabric(info->fabric_attr, &ofi.fabric, NULL));
OFICHKERR(fi_domain(ofi.fabric, info, &ofi.domain, NULL));
rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;
while (rx_ctx_cnt >> ++rx_ctx_bits);
av_attr.rx_ctx_bits = rx_ctx_bits;
av_attr.type = FI_AV_TABLE;
av_attr.count = chpl_numNodes;
OFICHKERR(fi_av_open(ofi.domain, &av_attr, &ofi.av, NULL));
OFICHKERR(fi_scalable_ep(ofi.domain, info, &ofi.ep, NULL));
OFICHKERR(fi_scalable_ep_bind(ofi.ep, &ofi.av->fid, 0));
/* set up tx and rx contexts */
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.size = 1024; /* ??? */
cq_attr.wait_obj = FI_WAIT_UNSPEC;
ofi.tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_tx_ctx,
sizeof(ofi.tx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_tx_ctx,
sizeof(ofi.tx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_tx_ctx; i++) {
OFICHKERR(fi_tx_context(ofi.ep, i, NULL, &ofi.tx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.tx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.tx_ep[i], &ofi.tx_cq[i]->fid, FI_TRANSMIT));
OFICHKERR(fi_enable(ofi.tx_ep[i]));
}
ofi.rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_rx_ctx,
sizeof(ofi.rx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_rx_ctx,
sizeof(ofi.rx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_rx_ctx; i++) {
OFICHKERR(fi_rx_context(ofi.ep, i, NULL, &ofi.rx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.rx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.rx_ep[i], &ofi.rx_cq[i]->fid, FI_RECV));
OFICHKERR(fi_enable(ofi.rx_ep[i]));
}
ofi.am_tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_tx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.am_tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_tx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
/* set up AM contexts */
for (i = 0; i < ofi.num_am_ctx; i++) {
OFICHKERR(fi_tx_context(ofi.ep, i+ofi.num_tx_ctx, NULL, &ofi.am_tx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_tx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.am_tx_ep[i], &ofi.am_tx_cq[i]->fid, FI_TRANSMIT));
OFICHKERR(fi_enable(ofi.am_tx_ep[i]));
}
ofi.am_rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_rx_ep[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
ofi.am_rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
sizeof(ofi.am_rx_cq[0]),
CHPL_RT_MD_COMM_PER_LOC_INFO,
0, 0);
for (i = 0; i < ofi.num_am_ctx; i++) {
OFICHKERR(fi_rx_context(ofi.ep, i+ofi.num_rx_ctx, NULL, &ofi.am_rx_ep[i], NULL));
OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_rx_cq[i], NULL));
OFICHKERR(fi_ep_bind(ofi.am_rx_ep[i], &ofi.am_rx_cq[i]->fid, FI_RECV));
OFICHKERR(fi_enable(ofi.am_rx_ep[i]));
}
OFICHKERR(fi_enable(ofi.ep));
libfabric_init_addrvec(rx_ctx_cnt, rx_ctx_bits);
OFICHKERR(fi_mr_reg(ofi.domain, 0, SIZE_MAX,
FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE |
FI_SEND | FI_RECV, 0,
(uint64_t) chpl_nodeID, 0, &ofi.mr, NULL));
fi_freeinfo(info); /* No error returned */
fi_freeinfo(hints); /* No error returned */
chpl_msg(2, "%d: completed libfabric initialization\n", chpl_nodeID);
}
