static int psmx2_getinfo(uint32_t version, const char *node,
const char *service, uint64_t flags,
struct fi_info *hints, struct fi_info **info)
{
struct fi_info *psmx2_info;
uint32_t cnt = 0;
void *dest_addr = NULL;
int ep_type = FI_EP_RDM;
int av_type = FI_AV_UNSPEC;
uint64_t mode = FI_CONTEXT;
enum fi_mr_mode mr_mode = FI_MR_SCALABLE;
enum fi_threading threading = FI_THREAD_COMPLETION;
enum fi_progress control_progress = FI_PROGRESS_MANUAL;
enum fi_progress data_progress = FI_PROGRESS_MANUAL;
uint64_t caps = PSMX2_CAPS;
uint64_t max_tag_value = -1ULL;
int err = -FI_ENODATA;
FI_INFO(&psmx2_prov, FI_LOG_CORE,"\n");
*info = NULL;
if (psm2_ep_num_devunits(&cnt) || !cnt) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"no PSM device is found.\n");
return -FI_ENODATA;
}
psmx2_init_env();
if (node && !(flags & FI_SOURCE)) {
dest_addr = psmx2_resolve_name(node, 0);
if (dest_addr)
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"node '%s' resolved to <epid=0x%llx, vl=%d>\n", node,
((struct psmx2_ep_name *)dest_addr)->epid,
((struct psmx2_ep_name *)dest_addr)->vlane);
else
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"failed to resolve node '%s'.\n", node);
}
if (hints) {
switch (hints->addr_format) {
case FI_FORMAT_UNSPEC:
case FI_ADDR_PSMX:
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->addr_format=%d, supported=%d,%d.\n",
hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX);
goto err_out;
}
if (hints->ep_attr) {
switch (hints->ep_attr->type) {
case FI_EP_UNSPEC:
case FI_EP_DGRAM:
case FI_EP_RDM:
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->ep_attr->type=%d, supported=%d,%d,%d.\n",
hints->ep_attr->type, FI_EP_UNSPEC,
FI_EP_DGRAM, FI_EP_RDM);
goto err_out;
}
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_PSMX:
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->protocol=%d, supported=%d %d\n",
hints->ep_attr->protocol,
FI_PROTO_UNSPEC, FI_PROTO_PSMX);
goto err_out;
}
if (hints->ep_attr->tx_ctx_cnt > 1) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->tx_ctx_cnt);
goto err_out;
}
if (hints->ep_attr->rx_ctx_cnt > 1) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->rx_ctx_cnt);
goto err_out;
}
}
if ((hints->caps & PSMX2_CAPS) != hints->caps) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->caps=0x%llx, supported=0x%llx\n",
hints->caps, PSMX2_CAPS);
goto err_out;
}
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & PSMX2_OP_FLAGS) !=
hints->tx_attr->op_flags) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx->flags=0x%llx, "
"supported=0x%llx\n",
hints->tx_attr->op_flags,
PSMX2_OP_FLAGS);
goto err_out;
}
if (hints->tx_attr->inject_size > PSMX2_INJECT_SIZE) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->inject_size=%ld,"
"supported=%ld.\n",
hints->tx_attr->inject_size,
PSMX2_INJECT_SIZE);
goto err_out;
}
}
if (hints->rx_attr &&
(hints->rx_attr->op_flags & PSMX2_OP_FLAGS) !=
hints->rx_attr->op_flags) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->rx->flags=0x%llx, supported=0x%llx\n",
hints->rx_attr->op_flags, PSMX2_OP_FLAGS);
goto err_out;
}
if ((hints->caps & FI_TAGGED) || (hints->caps & FI_MSG)) {
if ((hints->mode & FI_CONTEXT) != FI_CONTEXT) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->mode=0x%llx, required=0x%llx\n",
hints->mode, FI_CONTEXT);
goto err_out;
}
}
else {
mode = 0;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strcmp(hints->fabric_attr->name, PSMX2_FABRIC_NAME)) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->fabric_name=%s, supported=psm\n",
hints->fabric_attr->name);
goto err_out;
}
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strcmp(hints->domain_attr->name, PSMX2_DOMAIN_NAME)) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_name=%s, supported=psm\n",
hints->domain_attr->name);
goto err_out;
}
switch (hints->domain_attr->av_type) {
case FI_AV_UNSPEC:
case FI_AV_MAP:
case FI_AV_TABLE:
av_type = hints->domain_attr->av_type;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_attr->av_type=%d, supported=%d %d %d\n",
hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP,
FI_AV_TABLE);
goto err_out;
}
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
break;
case FI_MR_BASIC:
case FI_MR_SCALABLE:
mr_mode = hints->domain_attr->mr_mode;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_attr->mr_mode=%d, supported=%d %d %d\n",
hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC,
FI_MR_SCALABLE);
goto err_out;
}
switch (hints->domain_attr->threading) {
case FI_THREAD_UNSPEC:
break;
case FI_THREAD_FID:
case FI_THREAD_ENDPOINT:
case FI_THREAD_COMPLETION:
case FI_THREAD_DOMAIN:
threading = hints->domain_attr->threading;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_attr->threading=%d, supported=%d %d %d %d %d\n",
hints->domain_attr->threading, FI_THREAD_UNSPEC,
FI_THREAD_FID, FI_THREAD_ENDPOINT, FI_THREAD_COMPLETION,
FI_THREAD_DOMAIN);
goto err_out;
}
switch (hints->domain_attr->control_progress) {
case FI_PROGRESS_UNSPEC:
break;
case FI_PROGRESS_MANUAL:
case FI_PROGRESS_AUTO:
control_progress = hints->domain_attr->control_progress;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_attr->control_progress=%d, supported=%d %d %d\n",
hints->domain_attr->control_progress, FI_PROGRESS_UNSPEC,
FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO);
goto err_out;
}
switch (hints->domain_attr->data_progress) {
case FI_PROGRESS_UNSPEC:
break;
case FI_PROGRESS_MANUAL:
case FI_PROGRESS_AUTO:
data_progress = hints->domain_attr->data_progress;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->domain_attr->data_progress=%d, supported=%d %d %d\n",
hints->domain_attr->data_progress, FI_PROGRESS_UNSPEC,
FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO);
goto err_out;
}
}
if (hints->ep_attr) {
if (hints->ep_attr->max_msg_size > PSMX2_MAX_MSG_SIZE) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->ep_attr->max_msg_size=%ld,"
"supported=%ld.\n",
hints->ep_attr->max_msg_size,
PSMX2_MAX_MSG_SIZE);
goto err_out;
}
max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format);
}
if (hints->tx_attr) {
if ((hints->tx_attr->msg_order & PSMX2_MSG_ORDER) !=
hints->tx_attr->msg_order) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->tx_attr->msg_order,
PSMX2_MSG_ORDER);
goto err_out;
}
if ((hints->tx_attr->comp_order & PSMX2_COMP_ORDER) !=
hints->tx_attr->comp_order) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->tx_attr->comp_order,
PSMX2_COMP_ORDER);
goto err_out;
}
if (hints->tx_attr->inject_size > PSMX2_INJECT_SIZE) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->inject_size=%ld,"
"supported=%d.\n",
hints->tx_attr->inject_size,
PSMX2_INJECT_SIZE);
goto err_out;
}
if (hints->tx_attr->iov_limit > 1) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->iov_limit=%ld,"
"supported=1.\n",
hints->tx_attr->iov_limit);
goto err_out;
}
if (hints->tx_attr->rma_iov_limit > 1) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->tx_attr->rma_iov_limit=%ld,"
"supported=1.\n",
hints->tx_attr->rma_iov_limit);
goto err_out;
}
}
if (hints->rx_attr) {
if ((hints->rx_attr->msg_order & PSMX2_MSG_ORDER) !=
hints->rx_attr->msg_order) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->rx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->rx_attr->msg_order,
PSMX2_MSG_ORDER);
goto err_out;
}
if ((hints->rx_attr->comp_order & PSMX2_COMP_ORDER) !=
hints->rx_attr->comp_order) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->rx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->rx_attr->comp_order,
PSMX2_COMP_ORDER);
goto err_out;
}
if (hints->rx_attr->iov_limit > 1) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"hints->rx_attr->iov_limit=%ld,"
"supported=1.\n",
hints->rx_attr->iov_limit);
goto err_out;
}
}
if (hints->caps)
caps = hints->caps;
/* TODO: check other fields of hints */
}
psmx2_info = fi_allocinfo();
if (!psmx2_info) {
err = -FI_ENOMEM;
goto err_out;
}
psmx2_info->ep_attr->type = ep_type;
psmx2_info->ep_attr->protocol = FI_PROTO_PSMX;
psmx2_info->ep_attr->protocol_version = PSM2_VERNO;
psmx2_info->ep_attr->max_msg_size = PSMX2_MAX_MSG_SIZE;
psmx2_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value);
psmx2_info->ep_attr->tx_ctx_cnt = 1;
psmx2_info->ep_attr->rx_ctx_cnt = 1;
psmx2_info->domain_attr->threading = threading;
psmx2_info->domain_attr->control_progress = control_progress;
psmx2_info->domain_attr->data_progress = data_progress;
psmx2_info->domain_attr->name = strdup(PSMX2_DOMAIN_NAME);
psmx2_info->domain_attr->resource_mgmt = FI_RM_ENABLED;
psmx2_info->domain_attr->av_type = av_type;
psmx2_info->domain_attr->mr_mode = mr_mode;
psmx2_info->domain_attr->mr_key_size = sizeof(uint64_t);
psmx2_info->domain_attr->cq_data_size = 4;
psmx2_info->domain_attr->cq_cnt = 65535;
psmx2_info->domain_attr->ep_cnt = 65535;
psmx2_info->domain_attr->tx_ctx_cnt = 1;
psmx2_info->domain_attr->rx_ctx_cnt = 1;
psmx2_info->domain_attr->max_ep_tx_ctx = 65535;
psmx2_info->domain_attr->max_ep_rx_ctx = 1;
psmx2_info->domain_attr->max_ep_stx_ctx = 65535;
psmx2_info->domain_attr->max_ep_srx_ctx = 0;
psmx2_info->next = NULL;
psmx2_info->caps = caps;
psmx2_info->mode = mode;
psmx2_info->addr_format = FI_ADDR_PSMX;
psmx2_info->src_addrlen = 0;
psmx2_info->dest_addrlen = sizeof(struct psmx2_ep_name);
psmx2_info->src_addr = NULL;
psmx2_info->dest_addr = dest_addr;
psmx2_info->fabric_attr->name = strdup(PSMX2_FABRIC_NAME);
psmx2_info->fabric_attr->prov_name = NULL;
psmx2_info->tx_attr->caps = psmx2_info->caps;
psmx2_info->tx_attr->mode = psmx2_info->mode;
psmx2_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : 0;
psmx2_info->tx_attr->msg_order = PSMX2_MSG_ORDER;
psmx2_info->tx_attr->comp_order = PSMX2_COMP_ORDER;
psmx2_info->tx_attr->inject_size = PSMX2_INJECT_SIZE;
psmx2_info->tx_attr->size = UINT64_MAX;
psmx2_info->tx_attr->iov_limit = 1;
psmx2_info->tx_attr->rma_iov_limit = 1;
psmx2_info->rx_attr->caps = psmx2_info->caps;
psmx2_info->rx_attr->mode = psmx2_info->mode;
psmx2_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->rx_attr->op_flags)
? hints->rx_attr->op_flags : 0;
psmx2_info->rx_attr->msg_order = PSMX2_MSG_ORDER;
psmx2_info->rx_attr->comp_order = PSMX2_COMP_ORDER;
psmx2_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */
psmx2_info->rx_attr->size = UINT64_MAX;
psmx2_info->rx_attr->iov_limit = 1;
*info = psmx2_info;
return 0;
err_out:
if (dest_addr)
free(dest_addr);
return err;
}
static int psmx_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct fi_info *psmx_info;
uint32_t cnt = 0;
void *dest_addr = NULL;
int ep_type = FI_EP_RDM;
int av_type = FI_AV_UNSPEC;
enum fi_mr_mode mr_mode = FI_MR_SCALABLE;
int caps = 0;
uint64_t max_tag_value = 0;
int err = -FI_ENODATA;
FI_INFO(&psmx_prov, FI_LOG_CORE,"\n");
*info = NULL;
if (psm_ep_num_devunits(&cnt) || !cnt) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"no PSM device is found.\n");
return -FI_ENODATA;
}
if (node && !(flags & FI_SOURCE))
dest_addr = psmx_resolve_name(node, 0);
if (hints) {
switch (hints->addr_format) {
case FI_FORMAT_UNSPEC:
case FI_ADDR_PSMX:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->addr_format=%d, supported=%d,%d.\n",
hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX);
goto err_out;
}
if (hints->ep_attr) {
switch (hints->ep_attr->type) {
case FI_EP_UNSPEC:
case FI_EP_DGRAM:
case FI_EP_RDM:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->type=%d, supported=%d,%d,%d.\n",
hints->ep_attr->type, FI_EP_UNSPEC,
FI_EP_DGRAM, FI_EP_RDM);
goto err_out;
}
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_PSMX:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->protocol=%d, supported=%d %d\n",
hints->ep_attr->protocol,
FI_PROTO_UNSPEC, FI_PROTO_PSMX);
goto err_out;
}
if (hints->ep_attr->tx_ctx_cnt > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->tx_ctx_cnt);
goto err_out;
}
if (hints->ep_attr->rx_ctx_cnt > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->rx_ctx_cnt);
goto err_out;
}
}
if ((hints->caps & PSMX_CAPS) != hints->caps &&
(hints->caps & PSMX_CAPS2) != hints->caps) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->caps=0x%llx, supported=0x%llx,0x%llx\n",
hints->caps, PSMX_CAPS, PSMX_CAPS2);
goto err_out;
}
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & PSMX_OP_FLAGS) !=
hints->tx_attr->op_flags) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx->flags=0x%llx, "
"supported=0x%llx\n",
hints->tx_attr->op_flags,
PSMX_OP_FLAGS);
goto err_out;
}
if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->inject_size=%ld,"
"supported=%ld.\n",
hints->tx_attr->inject_size,
PSMX_INJECT_SIZE);
goto err_out;
}
}
if (hints->rx_attr &&
(hints->rx_attr->op_flags & PSMX_OP_FLAGS) !=
hints->rx_attr->op_flags) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->rx->flags=0x%llx, supported=0x%llx\n",
hints->rx_attr->op_flags, PSMX_OP_FLAGS);
goto err_out;
}
if ((hints->mode & PSMX_MODE) != PSMX_MODE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->mode=0x%llx, required=0x%llx\n",
hints->mode, PSMX_MODE);
goto err_out;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, PSMX_FABRIC_NAME, PSMX_FABRIC_NAME_LEN)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->fabric_name=%s, supported=psm\n",
hints->fabric_attr->name);
goto err_out;
}
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strncmp(hints->domain_attr->name, PSMX_DOMAIN_NAME,
PSMX_DOMAIN_NAME_LEN)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_name=%s, supported=psm\n",
hints->domain_attr->name);
goto err_out;
}
switch (hints->domain_attr->av_type) {
case FI_AV_UNSPEC:
case FI_AV_MAP:
case FI_AV_TABLE:
av_type = hints->domain_attr->av_type;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->av_type=%d, supported=%d %d %d\n",
hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP,
FI_AV_TABLE);
goto err_out;
}
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
break;
case FI_MR_BASIC:
case FI_MR_SCALABLE:
mr_mode = hints->domain_attr->mr_mode;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->mr_mode=%d, supported=%d %d %d\n",
hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC,
FI_MR_SCALABLE);
goto err_out;
}
}
if (hints->ep_attr) {
if (hints->ep_attr->max_msg_size > PSMX_MAX_MSG_SIZE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->max_msg_size=%ld,"
"supported=%ld.\n",
hints->ep_attr->max_msg_size,
PSMX_MAX_MSG_SIZE);
goto err_out;
}
max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format);
}
caps = hints->caps;
/* TODO: check other fields of hints */
}
if (psmx_reserve_tag_bits(&caps, &max_tag_value) < 0)
goto err_out;
psmx_info = fi_allocinfo();
if (!psmx_info) {
err = -FI_ENOMEM;
goto err_out;
}
psmx_info->ep_attr->type = ep_type;
psmx_info->ep_attr->protocol = FI_PROTO_PSMX;
psmx_info->ep_attr->protocol_version = PSM_VERNO;
psmx_info->ep_attr->max_msg_size = PSMX_MAX_MSG_SIZE;
psmx_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value);
psmx_info->ep_attr->tx_ctx_cnt = 1;
psmx_info->ep_attr->rx_ctx_cnt = 1;
psmx_info->domain_attr->threading = FI_THREAD_COMPLETION;
psmx_info->domain_attr->control_progress = FI_PROGRESS_MANUAL;
psmx_info->domain_attr->data_progress = FI_PROGRESS_MANUAL;
psmx_info->domain_attr->name = strdup(PSMX_DOMAIN_NAME);
psmx_info->domain_attr->resource_mgmt = FI_RM_ENABLED;
psmx_info->domain_attr->av_type = av_type;
psmx_info->domain_attr->mr_mode = mr_mode;
psmx_info->domain_attr->mr_key_size = sizeof(uint64_t);
psmx_info->domain_attr->cq_data_size = 4;
psmx_info->domain_attr->cq_cnt = 65535;
psmx_info->domain_attr->ep_cnt = 65535;
psmx_info->domain_attr->tx_ctx_cnt = 1;
psmx_info->domain_attr->rx_ctx_cnt = 1;
psmx_info->domain_attr->max_ep_tx_ctx = 65535;
psmx_info->domain_attr->max_ep_rx_ctx = 1;
psmx_info->domain_attr->max_ep_stx_ctx = 65535;
psmx_info->domain_attr->max_ep_srx_ctx = 0;
psmx_info->next = NULL;
psmx_info->caps = (hints && hints->caps) ? hints->caps : caps;
psmx_info->mode = PSMX_MODE;
psmx_info->addr_format = FI_ADDR_PSMX;
psmx_info->src_addrlen = 0;
psmx_info->dest_addrlen = sizeof(psm_epid_t);
psmx_info->src_addr = NULL;
psmx_info->dest_addr = dest_addr;
psmx_info->fabric_attr->name = strdup(PSMX_FABRIC_NAME);
psmx_info->fabric_attr->prov_name = strdup(PSMX_PROV_NAME);
psmx_info->tx_attr->caps = psmx_info->caps;
psmx_info->tx_attr->mode = psmx_info->mode;
psmx_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : 0;
psmx_info->tx_attr->msg_order = FI_ORDER_SAS;
psmx_info->tx_attr->comp_order = FI_ORDER_NONE;
psmx_info->tx_attr->inject_size = PSMX_INJECT_SIZE;
psmx_info->tx_attr->size = UINT64_MAX;
psmx_info->tx_attr->iov_limit = 1;
psmx_info->rx_attr->caps = psmx_info->caps;
psmx_info->rx_attr->mode = psmx_info->mode;
psmx_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : 0;
psmx_info->rx_attr->msg_order = FI_ORDER_SAS;
psmx_info->rx_attr->comp_order = FI_ORDER_NONE;
psmx_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */
psmx_info->rx_attr->size = UINT64_MAX;
psmx_info->rx_attr->iov_limit = 1;
*info = psmx_info;
return 0;
err_out:
return err;
}
static int psmx_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct fi_info *psmx_info;
uint32_t cnt = 0;
psm_epid_t *dest_addr = NULL;
struct psmx_src_name *src_addr;
int ep_type = FI_EP_RDM;
int av_type = FI_AV_UNSPEC;
uint64_t mode = FI_CONTEXT;
enum fi_mr_mode mr_mode = FI_MR_SCALABLE;
enum fi_threading threading = FI_THREAD_COMPLETION;
enum fi_progress control_progress = FI_PROGRESS_MANUAL;
enum fi_progress data_progress = FI_PROGRESS_MANUAL;
int caps = 0;
uint64_t max_tag_value = 0;
int err = -FI_ENODATA;
int svc0, svc = PSMX_ANY_SERVICE;
FI_INFO(&psmx_prov, FI_LOG_CORE,"\n");
*info = NULL;
if (psmx_init_lib())
return -FI_ENODATA;
if (psm_ep_num_devunits(&cnt) || !cnt) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"no PSM device is found.\n");
return -FI_ENODATA;
}
psmx_init_env();
src_addr = calloc(1, sizeof(*src_addr));
if (!src_addr) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"failed to allocate src addr.\n");
return -FI_ENODATA;
}
src_addr->unit = PSMX_DEFAULT_UNIT;
src_addr->port = PSMX_DEFAULT_PORT;
src_addr->service = PSMX_ANY_SERVICE;
if (flags & FI_SOURCE) {
if (node)
sscanf(node, "%*[^:]:%d:%d", &src_addr->unit, &src_addr->port);
if (service)
sscanf(service, "%d", &src_addr->service);
FI_INFO(&psmx_prov, FI_LOG_CORE,
"node '%s' service '%s' converted to <unit=%d, port=%d, service=%d>\n",
node, service, src_addr->unit, src_addr->port, src_addr->service);
} else if (node) {
if (service)
svc = atoi(service);
svc0 = svc;
dest_addr = psmx_ns_resolve_name(node, &svc);
if (dest_addr) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"'%s:%u' resolved to <epid=0x%llx>:%u\n", node, svc0,
*dest_addr, svc);
} else {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"failed to resolve '%s:%u'.\n", node, svc);
err = -FI_ENODATA;
goto err_out;
}
}
if (hints) {
switch (hints->addr_format) {
case FI_FORMAT_UNSPEC:
case FI_ADDR_PSMX:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->addr_format=%d, supported=%d,%d.\n",
hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX);
goto err_out;
}
if (hints->ep_attr) {
switch (hints->ep_attr->type) {
case FI_EP_UNSPEC:
case FI_EP_DGRAM:
case FI_EP_RDM:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->type=%d, supported=%d,%d,%d.\n",
hints->ep_attr->type, FI_EP_UNSPEC,
FI_EP_DGRAM, FI_EP_RDM);
goto err_out;
}
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_PSMX:
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->protocol=%d, supported=%d %d\n",
hints->ep_attr->protocol,
FI_PROTO_UNSPEC, FI_PROTO_PSMX);
goto err_out;
}
if (hints->ep_attr->tx_ctx_cnt > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->tx_ctx_cnt);
goto err_out;
}
if (hints->ep_attr->rx_ctx_cnt > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n",
hints->ep_attr->rx_ctx_cnt);
goto err_out;
}
}
if ((hints->caps & PSMX_CAPS) != hints->caps &&
(hints->caps & PSMX_CAPS2) != hints->caps) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->caps=0x%llx, supported=0x%llx,0x%llx\n",
hints->caps, PSMX_CAPS, PSMX_CAPS2);
goto err_out;
}
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & PSMX_OP_FLAGS) !=
hints->tx_attr->op_flags) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx->flags=0x%llx, "
"supported=0x%llx\n",
hints->tx_attr->op_flags,
PSMX_OP_FLAGS);
goto err_out;
}
if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->inject_size=%ld,"
"supported=%ld.\n",
hints->tx_attr->inject_size,
PSMX_INJECT_SIZE);
goto err_out;
}
}
if (hints->rx_attr &&
(hints->rx_attr->op_flags & PSMX_OP_FLAGS) !=
hints->rx_attr->op_flags) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->rx->flags=0x%llx, supported=0x%llx\n",
hints->rx_attr->op_flags, PSMX_OP_FLAGS);
goto err_out;
}
if ((hints->caps & FI_TAGGED) ||
((hints->caps & FI_MSG) && !psmx_env.am_msg)) {
if ((hints->mode & FI_CONTEXT) != FI_CONTEXT) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->mode=0x%llx, required=0x%llx\n",
hints->mode, FI_CONTEXT);
goto err_out;
}
} else {
mode = 0;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strcmp(hints->fabric_attr->name, PSMX_FABRIC_NAME)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->fabric_name=%s, supported=psm\n",
hints->fabric_attr->name);
goto err_out;
}
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strcmp(hints->domain_attr->name, PSMX_DOMAIN_NAME)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_name=%s, supported=psm\n",
hints->domain_attr->name);
goto err_out;
}
switch (hints->domain_attr->av_type) {
case FI_AV_UNSPEC:
case FI_AV_MAP:
case FI_AV_TABLE:
av_type = hints->domain_attr->av_type;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->av_type=%d, supported=%d %d %d\n",
hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP,
FI_AV_TABLE);
goto err_out;
}
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
break;
case FI_MR_BASIC:
case FI_MR_SCALABLE:
mr_mode = hints->domain_attr->mr_mode;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->mr_mode=%d, supported=%d %d %d\n",
hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC,
FI_MR_SCALABLE);
goto err_out;
}
switch (hints->domain_attr->threading) {
case FI_THREAD_UNSPEC:
break;
case FI_THREAD_FID:
case FI_THREAD_ENDPOINT:
case FI_THREAD_COMPLETION:
case FI_THREAD_DOMAIN:
threading = hints->domain_attr->threading;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->threading=%d, supported=%d %d %d %d %d\n",
hints->domain_attr->threading, FI_THREAD_UNSPEC,
FI_THREAD_FID, FI_THREAD_ENDPOINT, FI_THREAD_COMPLETION,
FI_THREAD_DOMAIN);
goto err_out;
}
switch (hints->domain_attr->control_progress) {
case FI_PROGRESS_UNSPEC:
break;
case FI_PROGRESS_MANUAL:
case FI_PROGRESS_AUTO:
control_progress = hints->domain_attr->control_progress;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->control_progress=%d, supported=%d %d %d\n",
hints->domain_attr->control_progress, FI_PROGRESS_UNSPEC,
FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO);
goto err_out;
}
switch (hints->domain_attr->data_progress) {
case FI_PROGRESS_UNSPEC:
break;
case FI_PROGRESS_MANUAL:
case FI_PROGRESS_AUTO:
data_progress = hints->domain_attr->data_progress;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->data_progress=%d, supported=%d %d %d\n",
hints->domain_attr->data_progress, FI_PROGRESS_UNSPEC,
FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO);
goto err_out;
}
if (hints->domain_attr->caps & FI_SHARED_AV) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->domain_attr->caps=%lx, shared AV is unsupported\n",
hints->domain_attr->caps);
goto err_out;
}
}
if (hints->ep_attr) {
if (hints->ep_attr->max_msg_size > PSMX_MAX_MSG_SIZE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->ep_attr->max_msg_size=%ld,"
"supported=%ld.\n",
hints->ep_attr->max_msg_size,
PSMX_MAX_MSG_SIZE);
goto err_out;
}
max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format);
}
if (hints->tx_attr) {
if ((hints->tx_attr->msg_order & PSMX_MSG_ORDER) !=
hints->tx_attr->msg_order) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->tx_attr->msg_order,
PSMX_MSG_ORDER);
goto err_out;
}
if ((hints->tx_attr->comp_order & PSMX_COMP_ORDER) !=
hints->tx_attr->comp_order) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->tx_attr->comp_order,
PSMX_COMP_ORDER);
goto err_out;
}
if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->inject_size=%ld,"
"supported=%d.\n",
hints->tx_attr->inject_size,
PSMX_INJECT_SIZE);
goto err_out;
}
if (hints->tx_attr->iov_limit > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->iov_limit=%ld,"
"supported=1.\n",
hints->tx_attr->iov_limit);
goto err_out;
}
if (hints->tx_attr->rma_iov_limit > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->tx_attr->rma_iov_limit=%ld,"
"supported=1.\n",
hints->tx_attr->rma_iov_limit);
goto err_out;
}
}
if (hints->rx_attr) {
if ((hints->rx_attr->msg_order & PSMX_MSG_ORDER) !=
hints->rx_attr->msg_order) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->rx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->rx_attr->msg_order,
PSMX_MSG_ORDER);
goto err_out;
}
if ((hints->rx_attr->comp_order & PSMX_COMP_ORDER) !=
hints->rx_attr->comp_order) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->rx_attr->msg_order=%lx,"
"supported=%lx.\n",
hints->rx_attr->comp_order,
PSMX_COMP_ORDER);
goto err_out;
}
if (hints->rx_attr->iov_limit > 1) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"hints->rx_attr->iov_limit=%ld,"
"supported=1.\n",
hints->rx_attr->iov_limit);
goto err_out;
}
}
caps = hints->caps;
/* TODO: check other fields of hints */
}
if (psmx_reserve_tag_bits(&caps, &max_tag_value) < 0)
goto err_out;
psmx_info = fi_allocinfo();
if (!psmx_info) {
err = -FI_ENOMEM;
goto err_out;
}
psmx_info->ep_attr->type = ep_type;
psmx_info->ep_attr->protocol = FI_PROTO_PSMX;
psmx_info->ep_attr->protocol_version = PSM_VERNO;
psmx_info->ep_attr->max_msg_size = PSMX_MAX_MSG_SIZE;
psmx_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value);
psmx_info->ep_attr->tx_ctx_cnt = 1;
psmx_info->ep_attr->rx_ctx_cnt = 1;
psmx_info->domain_attr->threading = threading;
psmx_info->domain_attr->control_progress = control_progress;
psmx_info->domain_attr->data_progress = data_progress;
psmx_info->domain_attr->name = strdup(PSMX_DOMAIN_NAME);
psmx_info->domain_attr->resource_mgmt = FI_RM_ENABLED;
psmx_info->domain_attr->av_type = av_type;
psmx_info->domain_attr->mr_mode = mr_mode;
psmx_info->domain_attr->mr_key_size = sizeof(uint64_t);
psmx_info->domain_attr->cq_data_size = 4;
psmx_info->domain_attr->cq_cnt = 65535;
psmx_info->domain_attr->ep_cnt = 65535;
psmx_info->domain_attr->tx_ctx_cnt = 1;
psmx_info->domain_attr->rx_ctx_cnt = 1;
psmx_info->domain_attr->max_ep_tx_ctx = 1;
psmx_info->domain_attr->max_ep_rx_ctx = 1;
psmx_info->domain_attr->max_ep_stx_ctx = 65535;
psmx_info->domain_attr->max_ep_srx_ctx = 0;
psmx_info->domain_attr->cntr_cnt = 65535;
psmx_info->domain_attr->mr_iov_limit = 65535;
psmx_info->domain_attr->caps = PSMX_DOM_CAPS;
psmx_info->domain_attr->mode = 0;
psmx_info->next = NULL;
psmx_info->caps = (hints && hints->caps) ? hints->caps : caps;
psmx_info->mode = mode;
psmx_info->addr_format = FI_ADDR_PSMX;
psmx_info->src_addr = src_addr;
psmx_info->src_addrlen = sizeof(*src_addr);
psmx_info->dest_addr = dest_addr;
psmx_info->dest_addrlen = sizeof(*dest_addr);
psmx_info->fabric_attr->name = strdup(PSMX_FABRIC_NAME);
psmx_info->fabric_attr->prov_name = NULL;
psmx_info->fabric_attr->prov_version = PSMX_VERSION;
psmx_info->tx_attr->caps = psmx_info->caps;
psmx_info->tx_attr->mode = psmx_info->mode;
psmx_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags)
? hints->tx_attr->op_flags : 0;
psmx_info->tx_attr->msg_order = PSMX_MSG_ORDER;
psmx_info->tx_attr->comp_order = PSMX_COMP_ORDER;
psmx_info->tx_attr->inject_size = PSMX_INJECT_SIZE;
psmx_info->tx_attr->size = UINT64_MAX;
psmx_info->tx_attr->iov_limit = 1;
psmx_info->tx_attr->rma_iov_limit = 1;
psmx_info->rx_attr->caps = psmx_info->caps;
psmx_info->rx_attr->mode = psmx_info->mode;
psmx_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->rx_attr->op_flags)
? hints->rx_attr->op_flags : 0;
psmx_info->rx_attr->msg_order = PSMX_MSG_ORDER;
psmx_info->rx_attr->comp_order = PSMX_COMP_ORDER;
psmx_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */
psmx_info->rx_attr->size = UINT64_MAX;
psmx_info->rx_attr->iov_limit = 1;
*info = psmx_info;
return 0;
err_out:
free(dest_addr);
free(src_addr);
return err;
}
